diff --git a/app/channels/application_cable/connection.rb b/app/channels/application_cable/connection.rb
index af25058ad707f8107aa72d6a5983c06975e3ef4e..a427edb998d3f2c629e5f3a64db971bf591b67d4 100644
--- a/app/channels/application_cable/connection.rb
+++ b/app/channels/application_cable/connection.rb
@@ -1,21 +1,22 @@
module ApplicationCable
- ##
- # This class identifies the current user in a websocket communication using ApplicationCable
+ ##
+ # This class identifies the current user in a websocket communication using ApplicationCable
class Connection < ActionCable::Connection::Base
- identified_by :current_user
+ identified_by :current_user
- def connect
- self.current_user = find_user
- end
+ def connect
+ self.current_user = find_user
+ end
- def find_user
- user_id = cookies.signed["user.id"]
- current_user = User.find_by(id: user_id)
- if current_user
- current_user
- else
- reject_unauthorized_connection
- end
+ def find_user
+ current_user = User.find_by(labs_user_id: "2")
+ if current_user
+ puts "Current user case, id #{current_user.id}"
+ current_user
+ else
+ puts "Unauthorized connection rejected"
+ reject_unauthorized_connection
end
+ end
end
end
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 81692ba10881bc1382451b92cbf2931dbee1e43d..3863792f9c01b9642dfa79457ca3798a0e38b640 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -1,10 +1,9 @@
class ApplicationController < ActionController::Base
- include Authentication
-
- def send_file
- File.open("tmp/#{params[:filename]}", 'r') do |f|
- send_data f.read, type: "text/json", filename: params[:filename]
- end
- end
+ include Authentication
+ def send_file
+ File.open("tmp/#{params[:filename]}", "r") do |f|
+ send_data f.read, type: "text/json", filename: params[:filename]
+ end
+ end
end
diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb
index 2b7a4aadb88fc1af252e7cab37f6b438f9067e0f..ecd7a53bdef6c73b582507b83ac04b7cce08ea5c 100644
--- a/app/controllers/catalog_controller.rb
+++ b/app/controllers/catalog_controller.rb
@@ -1,220 +1,220 @@
class CatalogController < ApplicationController
+ before_action :authenticate_user!, :strip_input_fields
- before_action :authenticate_user!, :strip_input_fields
+ def home
+ end
- def home
-
- end
-
- ##
- # Creates a search query and submit it to the index. Retrieve and displays results + metadata.
- def index
- if params[:q]
- @search_type = params[:search_type].nil? ? "exact" : params[:search_type]
- @solr_params = SolrQuery.new(@search_type).to_params
- @solr_params[:q] = params[:q]
- @solr_params[:rows] = params[:per_page] if params[:per_page]
- @current_page = params[:page].to_i != 0 ? params[:page].to_i : 1
- @solr_params[:start] = params[:page].to_i != 0 ? @solr_params[:rows] * (params[:page].to_i-1) : 0
- @solr_params[:sort] = params[:sort] if params[:sort]
- if params[:f]
- params[:f].each do |k,v|
- if k == "date_created_dtsi" # v is a hash {to: "", from: ""}
- @solr_params[:fq] << "#{k}:[#{v['from']}T00:00:00Z TO #{v['to']}T00:00:00Z]"
- else
- if v.is_a? Array
- v.each do |val|
- @solr_params[:fq] << "#{k}:#{val}"
- end
- end
- end
- end
+ ##
+ # Creates a search query and submit it to the index. Retrieve and displays results + metadata.
+ def index
+ if params[:q]
+ @search_type = params[:search_type].nil? ? "exact" : params[:search_type]
+ @solr_params = SolrQuery.new(@search_type).to_params
+ @solr_params[:q] = params[:q]
+ @solr_params[:rows] = params[:per_page] if params[:per_page]
+ @current_page = params[:page].to_i != 0 ? params[:page].to_i : 1
+ @solr_params[:start] = params[:page].to_i != 0 ? @solr_params[:rows] * (params[:page].to_i - 1) : 0
+ @solr_params[:sort] = params[:sort] if params[:sort]
+ if params[:f]
+ params[:f].each do |k, v|
+ if k == "date_created_dtsi" # v is a hash {to: "", from: ""}
+ @solr_params[:fq] << "#{k}:[#{v["from"]}T00:00:00Z TO #{v["to"]}T00:00:00Z]"
+ else
+ if v.is_a? Array
+ v.each do |val|
+ @solr_params[:fq] << "#{k}:#{val}"
+ end
end
- session['search_params'] = @solr_params
- session['query_params'] = params.to_unsafe_h.slice('q', 'page', 'per_page','sort', 'f')
- @results = SolrSearcher.query @solr_params
- puts @results.to_json if Rails.env == "development"
- @resulting_docs = @results['response']['docs'].map do |solr_doc|
- case solr_doc['has_model_ssim']
- when ['Article']
- Article.from_solr_doc solr_doc
- when ['Issue']
- Issue.from_solr_doc solr_doc
- end
- end
- entities_fields = I18n.t("newspapers.solr_fields").values_at(:persons, :locations, :organisations, :human_productions)
- @entities_labels = []
- entities_fields.each do |entity_field|
- (@entities_labels << @results['facets'][entity_field]['buckets'].map{|ne| ne['val']}).flatten! if @results['facets'][entity_field]
- end
- @entities_labels = helpers.get_entity_label @entities_labels
+ end
+ end
+ end
+ session["search_params"] = @solr_params
+ session["query_params"] = params.to_unsafe_h.slice("q", "page", "per_page", "sort", "f")
+ @results = SolrSearcher.query @solr_params
+ # puts @results.to_json if Rails.env == "development"
+ @resulting_docs = @results["response"]["docs"].map do |solr_doc|
+ case solr_doc["has_model_ssim"]
+ when ["Article"]
+ Article.from_solr_doc solr_doc
+ when ["Issue"]
+ Issue.from_solr_doc solr_doc
end
+ end
+ entities_fields = I18n.t("newspapers.solr_fields").values_at(:persons, :locations, :organisations, :human_productions)
+ @entities_labels = []
+ entities_fields.each do |entity_field|
+ (@entities_labels << @results["facets"][entity_field]["buckets"].map { |ne| ne["val"] }).flatten! if @results["facets"][entity_field]
+ end
+ @entities_labels = helpers.get_entity_label @entities_labels
end
+ end
- ##
- # Display an issue
- def show
- @issue = Issue.from_solr params[:id], with_pages=true, with_articles=true
- session['named_entities'] = Issue.named_entities @issue.id
- session['named_entities_labels'] = helpers.get_linked_entities session['named_entities'].map{ |k,v| v.keys }.flatten.uniq
- end
+ ##
+ # Display an issue
+ def show
+ @issue = Issue.from_solr params[:id], with_pages = true, with_articles = true
+ session["named_entities"] = Issue.named_entities @issue.id
+ session["named_entities_labels"] = helpers.get_linked_entities session["named_entities"].map { |k, v| v.keys }.flatten.uniq
+ puts session.inspect
+ end
- ##
- # Retrieve named entities for a list of documents (issue and/or articles)
- def named_entities_for_docs
- named_entities = {LOC: {}, PER: {}, ORG: {}, HumanProd: {}}
- params[:docs_ids].each do |doc_id|
- if doc_id.index('_article_').nil?
- doc_named_entities = session['named_entities']
- else # if article, filter stored list
- doc_named_entities = session['named_entities'].map{ |ne_type, ne_list|
- [ne_type,ne_list.select{ |linked_id, namedentities|
- namedentities.any?{ |ne|
- ne['article_id_ssi'] == doc_id
- }
- }.map{ |k,v| [k,v.select{ |ne| ne['article_id_ssi'] == doc_id }] }.to_h]
- }.to_h
- end
- named_entities[:LOC] = named_entities[:LOC].merge(doc_named_entities[:LOC]) do |key,oldval,newval|
- oldval.concat newval
- end
- named_entities[:ORG] = named_entities[:ORG].merge(doc_named_entities[:ORG]) do |key,oldval,newval|
- oldval.concat newval
- end
- named_entities[:PER] = named_entities[:PER].merge(doc_named_entities[:PER]) do |key,oldval,newval|
- oldval.concat newval
- end
- named_entities[:HumanProd] = named_entities[:HumanProd].merge(doc_named_entities[:HumanProd]) do |key,oldval,newval|
- oldval.concat newval
- end
- end
- render partial: 'named_entities/named_entities', locals: {named_entities: named_entities, linked_entities: session['named_entities_labels']}
+ ##
+ # Retrieve named entities for a list of documents (issue and/or articles)
+ def named_entities_for_docs
+ named_entities = { LOC: {}, PER: {}, ORG: {}, HumanProd: {} }
+ params[:docs_ids].each do |doc_id|
+ if doc_id.index("_article_").nil?
+ doc_named_entities = session["named_entities"]
+ else # if article, filter stored list
+ doc_named_entities = session["named_entities"].map { |ne_type, ne_list|
+ [ne_type, ne_list.select { |linked_id, namedentities|
+ namedentities.any? { |ne|
+ ne["article_id_ssi"] == doc_id
+ }
+ }.map { |k, v| [k, v.select { |ne| ne["article_id_ssi"] == doc_id }] }.to_h]
+ }.to_h
+ end
+ named_entities[:LOC] = named_entities[:LOC].merge(doc_named_entities[:LOC]) do |key, oldval, newval|
+ oldval.concat newval
+ end
+ named_entities[:ORG] = named_entities[:ORG].merge(doc_named_entities[:ORG]) do |key, oldval, newval|
+ oldval.concat newval
+ end
+ named_entities[:PER] = named_entities[:PER].merge(doc_named_entities[:PER]) do |key, oldval, newval|
+ oldval.concat newval
+ end
+ named_entities[:HumanProd] = named_entities[:HumanProd].merge(doc_named_entities[:HumanProd]) do |key, oldval, newval|
+ oldval.concat newval
+ end
end
+ render partial: "named_entities/named_entities", locals: { named_entities: named_entities, linked_entities: session["named_entities_labels"] }
+ end
- ##
- # Retrieve named entities for a dataset
- def named_entities_for_dataset
- dataset = Dataset.find(params[:dataset_id])
- named_entities = dataset.named_entities
- named_entities_labels = helpers.get_linked_entities named_entities.map{ |k,v| v.keys }.flatten.uniq
- render partial: 'named_entities/named_entities', locals: {named_entities: named_entities, linked_entities: named_entities_labels}
- end
+ ##
+ # Retrieve named entities for a dataset
+ def named_entities_for_dataset
+ dataset = Dataset.find(params[:dataset_id])
+ named_entities = dataset.named_entities
+ named_entities_labels = helpers.get_linked_entities named_entities.map { |k, v| v.keys }.flatten.uniq
+ render partial: "named_entities/named_entities", locals: { named_entities: named_entities, linked_entities: named_entities_labels }
+ end
- ##
- # Retrieve and display paginated facets
- def paginate_facets
- out = {}
- if params[:field_name] != ""
- search_params = session['search_params']
- search_params['rows'] = 0
- search_params['json.facet'] = {"#{params[:field_name]}": {terms: {
- field: params[:field_name],
- limit: 15,
- numBuckets: true,
- offset: (params[:current_page].to_i-1) * 15}}}.to_json
- res = SolrSearcher.query search_params
- entities_labels = [res['facets'][params[:field_name]]['buckets'].map{|ne| ne['val']}]
- entities_labels = helpers.get_entity_label entities_labels
- facet_constraints = search_params['fq'].select { |fq| fq.split(':')[0] == params[:field_name] }.map{|fq| {label: params[:field_name], value: fq.split(':')[1]} }
- out[:facets_entries] = []
- res['facets'][params[:field_name]]['buckets'].each do |facet_entry|
- out[:facets_entries] << render_to_string(layout: false, partial: "facet_entry", locals: {
- entities_labels: entities_labels,
- facet_constraints: facet_constraints,
- field: params[:field_name],
- facet: facet_entry,
- index: params[:current_page].to_i,
- per_page: 15
- })
- end
- end
- out[:pagination] = render_to_string(layout: false, partial: 'facet_pagination', locals: {nb_pages: params[:nb_pages].to_i, current_page: params[:current_page].to_i})
- render json: out
+ ##
+ # Retrieve and display paginated facets
+ def paginate_facets
+ out = {}
+ if params[:field_name] != ""
+ search_params = session["search_params"]
+ search_params["rows"] = 0
+ search_params["json.facet"] = { "#{params[:field_name]}": { terms: {
+ field: params[:field_name],
+ limit: 15,
+ numBuckets: true,
+ offset: (params[:current_page].to_i - 1) * 15,
+ } } }.to_json
+ res = SolrSearcher.query search_params
+ entities_labels = [res["facets"][params[:field_name]]["buckets"].map { |ne| ne["val"] }]
+ entities_labels = helpers.get_entity_label entities_labels
+ facet_constraints = search_params["fq"].select { |fq| fq.split(":")[0] == params[:field_name] }.map { |fq| { label: params[:field_name], value: fq.split(":")[1] } }
+ out[:facets_entries] = []
+ res["facets"][params[:field_name]]["buckets"].each do |facet_entry|
+ out[:facets_entries] << render_to_string(layout: false, partial: "facet_entry", locals: {
+ entities_labels: entities_labels,
+ facet_constraints: facet_constraints,
+ field: params[:field_name],
+ facet: facet_entry,
+ index: params[:current_page].to_i,
+ per_page: 15,
+ })
+ end
end
+ out[:pagination] = render_to_string(layout: false, partial: "facet_pagination", locals: { nb_pages: params[:nb_pages].to_i, current_page: params[:current_page].to_i })
+ render json: out
+ end
- ##
- # Open modal for date frequencies histogram in wide format
- def wide_dates_histogram
- out = {}
- out[:modal_content] = render_to_string(layout: false, partial: "wide_dates_histogram")
- render json: out
- end
+ ##
+ # Open modal for date frequencies histogram in wide format
+ def wide_dates_histogram
+ out = {}
+ out[:modal_content] = render_to_string(layout: false, partial: "wide_dates_histogram")
+ render json: out
+ end
- ##
- # Open Modal to confirm the creation of a compound article
- def confirm_compound_creation
- out = {}
- out[:modal_content] = render_to_string(layout: false, partial: "confirm_compound_creation", locals: {article_parts: params[:article_parts]})
- render json: out
- end
+ ##
+ # Open Modal to confirm the creation of a compound article
+ def confirm_compound_creation
+ out = {}
+ out[:modal_content] = render_to_string(layout: false, partial: "confirm_compound_creation", locals: { article_parts: params[:article_parts] })
+ render json: out
+ end
- ##
- # Create a new compound article
- def create_compound
- compound = CompoundArticle.new
- compound.user = current_user
- compound.title = params[:title]
- compound.issue_id = params[:issue_id]
- issue = Issue.from_solr params[:issue_id]
- compound.newspaper = issue.newspaper
- compound.date_created = issue.date_created
- compound.thumbnail_url = issue.thumbnail_url
- compound.language = issue.language
- compound.all_text = params[:all_text]
- compound.parts = params[:article_parts_ids]
- begin
- compound.save!
- render json: {status: 'ok', html: render_to_string(layout: false, partial: "compound_articles_panel", locals: {issue_id: params[:issue_id]})}
- rescue ActiveRecord::RecordNotUnique
- render json: {status: "error", message: "A compound article with this title already exists."}
- rescue ActiveRecord::RecordInvalid
- render json: {status: "error", message: "The title should not be blank."}
- end
+ ##
+ # Create a new compound article
+ def create_compound
+ compound = CompoundArticle.new
+ compound.user = current_user
+ compound.title = params[:title]
+ compound.issue_id = params[:issue_id]
+ issue = Issue.from_solr params[:issue_id]
+ compound.newspaper = issue.newspaper
+ compound.date_created = issue.date_created
+ compound.thumbnail_url = issue.thumbnail_url
+ compound.language = issue.language
+ compound.all_text = params[:all_text]
+ compound.parts = params[:article_parts_ids]
+ begin
+ compound.save!
+ render json: { status: "ok", html: render_to_string(layout: false, partial: "compound_articles_panel", locals: { issue_id: params[:issue_id] }) }
+ rescue ActiveRecord::RecordNotUnique
+ render json: { status: "error", message: "A compound article with this title already exists." }
+ rescue ActiveRecord::RecordInvalid
+ render json: { status: "error", message: "The title should not be blank." }
end
+ end
- ##
- # Delete an existing compound
- def delete_compound
- compound = CompoundArticle.find(params[:compound_id])
- issue_id = compound.issue_id
- current_user.datasets.each do |dataset|
- if dataset.documents.any?{|doc| doc['id'].to_s == compound.id.to_s}
- dataset.documents = dataset.documents.select{|doc| doc['id'].to_s != compound.id.to_s}
- dataset.save!
- end
- end
- compound.destroy
- out = {}
- out[:html] = render_to_string(layout: false, partial: "compound_articles_panel", locals: {issue_id: issue_id})
- out[:datasets] = render_to_string(layout: false, partial: "manage_datasets_content_show_page")
- render json: out
+ ##
+ # Delete an existing compound
+ def delete_compound
+ compound = CompoundArticle.find(params[:compound_id])
+ issue_id = compound.issue_id
+ current_user.datasets.each do |dataset|
+ if dataset.documents.any? { |doc| doc["id"].to_s == compound.id.to_s }
+ dataset.documents = dataset.documents.select { |doc| doc["id"].to_s != compound.id.to_s }
+ dataset.save!
+ end
end
+ compound.destroy
+ out = {}
+ out[:html] = render_to_string(layout: false, partial: "compound_articles_panel", locals: { issue_id: issue_id })
+ out[:datasets] = render_to_string(layout: false, partial: "manage_datasets_content_show_page")
+ render json: out
+ end
- ##
- # Retrieve and display a random sample of the result of a search
- def random_sample
- search_params = session['search_params'].with_indifferent_access
- search_params[:fq] = search_params[:fq].select {|elt| !elt.start_with? "has_model_ssim:" } if search_params[:fq]
- search_params[:fq] ||= []
- search_params[:fq] << "has_model_ssim:Article"
- search_params[:sort] = "rand#{(0...8).map { (65 + rand(26)).chr }.join} asc"
- results = SolrSearcher.query search_params
- results = results['response']['docs'].map do |solr_doc|
- case solr_doc['has_model_ssim']
- when ['Article']
- Article.from_solr_doc solr_doc
- when ['Issue']
- Issue.from_solr_doc solr_doc
- end
- end
- render json: {content: render_to_string(layout: false, partial: "random_sample", locals: {resulting_docs: results}) }
+ ##
+ # Retrieve and display a random sample of the result of a search
+ def random_sample
+ search_params = session["search_params"].with_indifferent_access
+ search_params[:fq] = search_params[:fq].select { |elt| !elt.start_with? "has_model_ssim:" } if search_params[:fq]
+ search_params[:fq] ||= []
+ search_params[:fq] << "has_model_ssim:Article"
+ search_params[:sort] = "rand#{(0...8).map { (65 + rand(26)).chr }.join} asc"
+ results = SolrSearcher.query search_params
+ results = results["response"]["docs"].map do |solr_doc|
+ case solr_doc["has_model_ssim"]
+ when ["Article"]
+ Article.from_solr_doc solr_doc
+ when ["Issue"]
+ Issue.from_solr_doc solr_doc
+ end
end
+ render json: { content: render_to_string(layout: false, partial: "random_sample", locals: { resulting_docs: results }) }
+ end
- private
+ private
- def strip_input_fields
- params.each do |key, value|
- params[key] = value.strip if value.respond_to?("strip")
- end
+ def strip_input_fields
+ params.each do |key, value|
+ params[key] = value.strip if value.respond_to?("strip")
end
+ end
end
diff --git a/app/controllers/dataset_controller.rb b/app/controllers/dataset_controller.rb
index e04f4b4e1265ad806aea318f755e5a70647f3a1f..ca0fbbceabd839850ae16861d9d0255ab0f08246 100644
--- a/app/controllers/dataset_controller.rb
+++ b/app/controllers/dataset_controller.rb
@@ -1,194 +1,203 @@
class DatasetController < ApplicationController
-
- before_action :authenticate_user!
-
- ##
- # List all datasets
- def index
- end
-
- ##
- # Display a single dataset
- def show
- @dataset = Dataset.find(params[:id])
- @current_page = params[:page] || 1
- @per_page = params[:per_page] || 10
- session[:working_dataset] = @dataset.id
- end
-
- ##
- # Create a new empty dataset
- def create_dataset
- dataset = Dataset.new
- dataset.user = current_user
- dataset.title = params[:title]
- begin
- dataset.save!
- render json: {status: 'ok'}
- rescue ActiveRecord::RecordNotUnique
- render json: {status: "error", message: "A dataset with this title already exists."}
- rescue ActiveRecord::RecordInvalid
- render json: {status: "error", message: "The title should not be blank."}
- end
- end
-
- ##
- # Rename an existing dataset
- def rename_dataset
- dataset = Dataset.find(params[:id])
- dataset.title = params[:title]
+ before_action :authenticate_user!
+
+ ##
+ # List all datasets
+ def index
+ puts "Listing datasets"
+ puts session.inspect
+ end
+
+ ##
+ # Display a single dataset
+ def show
+ puts "Finding dataset with id " + String(params[:id])
+ @dataset = Dataset.find(params[:id])
+ @current_page = params[:page] || 1
+ @per_page = params[:per_page] || 10
+ session[:working_dataset] = @dataset.id
+ puts "The session now has working_dataset " + String(session[:working_dataset])
+ end
+
+ ##
+ # Create a new empty dataset
+ def create_dataset
+ dataset = Dataset.new
+ dataset.user = current_user
+ dataset.title = params[:title]
+ begin
+ dataset.save!
+ render json: { status: "ok" }
+ rescue ActiveRecord::RecordNotUnique
+ render json: { status: "error", message: "A dataset with this title already exists." }
+ rescue ActiveRecord::RecordInvalid
+ render json: { status: "error", message: "The title should not be blank." }
+ end
+ end
+
+ ##
+ # Rename an existing dataset
+ def rename_dataset
+ dataset = Dataset.find(params[:id])
+ dataset.title = params[:title]
+ begin
+ dataset.save!
+ render json: { status: "ok" }
+ rescue ActiveRecord::RecordNotUnique
+ render json: { status: "error", message: "A dataset with this title already exists." }
+ rescue ActiveRecord::RecordInvalid
+ render json: { status: "error", message: "The title should not be blank." }
+ end
+ end
+
+ ##
+ # Import a public dataset
+ def import_dataset
+ to_copy = Dataset.find params[:original_dataset_id]
+ render json: { status: "error", message: "This dataset is not public." } unless to_copy.public?
+ new_dataset = Dataset.new
+ new_dataset.user_id = current_user.id
+ new_dataset.title = params[:title]
+ to_copy.documents.each do |doc|
+ if doc["type"] == "compound"
+ ca = CompoundArticle.find(doc["id"]).dup
+ ca.user = current_user
begin
- dataset.save!
- render json: {status: 'ok'}
+ ca.save!
rescue ActiveRecord::RecordNotUnique
- render json: {status: "error", message: "A dataset with this title already exists."}
- rescue ActiveRecord::RecordInvalid
- render json: {status: "error", message: "The title should not be blank."}
+ ca.title = "_#{(0...8).map { (65 + rand(26)).chr }.join}_#{ca.title}"
+ ca.save!
end
- end
-
- ##
- # Import a public dataset
- def import_dataset
- to_copy = Dataset.find params[:original_dataset_id]
- render json: {status: "error", message: "This dataset is not public."} unless to_copy.public?
- new_dataset = Dataset.new
- new_dataset.user_id = current_user.id
- new_dataset.title = params[:title]
- to_copy.documents.each do |doc|
- if doc['type'] == "compound"
- ca = CompoundArticle.find(doc['id']).dup
- ca.user = current_user
- begin
- ca.save!
- rescue ActiveRecord::RecordNotUnique
- ca.title = "_#{(0...8).map { (65 + rand(26)).chr }.join}_#{ca.title}"
- ca.save!
- end
- new_dataset.documents << {id: ca.id, type: "compound"}
- else
- new_dataset.documents << doc
- end
- end
- begin
- new_dataset.save!
- render json: {status: 'ok'}
- rescue ActiveRecord::RecordNotUnique
- render json: {status: "error", message: "A dataset with this title already exists."}
- rescue ActiveRecord::RecordInvalid
- render json: {status: "error", message: "The title should not be blank."}
- end
- end
-
- ##
- # Delete an existing dataset
- def delete_dataset
- dataset = Dataset.find(params[:dataset_id])
- dataset_id = dataset.id
- dataset.destroy
- if session[:working_dataset] == dataset_id
- if current_user.datasets.first
- session[:working_dataset] = current_user.datasets.first.id
- else
- session[:working_dataset] = nil
- end
- end
- end
-
- ##
- # Update the view of the list of datasets
- def update_datasets_list
- respond_to do |format|
- format.js
- end
- end
-
- ##
- #
- def set_working_dataset
- session[:working_dataset] = params[:dataset_id]
- @title = Dataset.find(session[:working_dataset]).title
- respond_to do |format|
- format.js
- end
- end
-
- def add_selected_documents
- out = {}
- @nb_added_docs = params[:documents_ids].size
- dataset = Dataset.find(session[:working_dataset])
- existing = dataset.add_documents params[:documents_ids] # Add docs and return existing ids
- @nb_added_docs -= existing.size
- title = dataset.title
- message = " #{@nb_added_docs} document#{@nb_added_docs > 1 ? "s were" : " was"} added to your dataset.
"
- message.concat "#{existing.size} document#{existing.size > 1 ? "s" : ""} already exist in this dataset.
" unless existing.empty?
- # render partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
- out['notif'] = render_to_string layout: false, partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
- out['nbissues'] = dataset.documents.select{|d| d['type'] == "issue" }.size
- out['nbarticles'] = dataset.documents.select{|d| d['type'] == "article" }.size
- out['nbdocs'] = out['nbissues'] + out['nbarticles']
- out['title'] = title
- out['results_datasets'] = params[:documents_ids].map{ |docid| [docid, render_to_string(layout: false, partial: 'catalog/result_datasets', locals: {doc_id: docid})] }.to_h
- render json: out
- end
-
- def add_compound
- out = {}
- dataset = Dataset.find(session[:working_dataset])
- existing = dataset.add_compound params[:compound_id] # Add docs and return existing ids
- title = dataset.title
- message = " The compound article was added to your dataset.
"
- out['notif'] = render_to_string layout: false, partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
- out['nbissues'] = dataset.documents.select{|d| d['type'] == "issue" }.size
- out['nbarticles'] = dataset.documents.select{|d| d['type'] == "article" }.size
- out['nbcompounds'] = dataset.documents.select{|d| d['type'] == "compound" }.size
- out['nbdocs'] = out['nbissues'] + out['nbarticles'] + out['nbcompounds']
- out['title'] = title
- render json: out
- end
-
- def remove_selected_documents
- @nb_removed_docs = params[:documents_ids].size
- dataset = Dataset.find(session[:working_dataset])
- dataset.remove_documents params[:documents_ids]
- redirect_to action: "show", id: dataset.id
- end
-
- def add_all_documents
- SearchToDatasetWorker.perform_async(current_user.id, session[:working_dataset], params[:search_params].to_unsafe_h)
- title = Dataset.find(session[:working_dataset]).title
- message = "Documents are being added to your dataset. You will be notified when the operation is done.
"
- render partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
- end
-
- def export_dataset
- ExportDatasetWorker.perform_async(current_user.id, params[:dataset_id], params[:export_type])
- title = Dataset.find(params[:dataset_id]).title
- message = "The export is being prepared. You will be notified when the operation is done.
"
- render partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
- end
-
- def toggle_sharing_status
- @dataset = Dataset.find(params[:dataset_id])
- @dataset.toggle!(:public)
- render partial: 'dataset_info'
- end
-
- def paginate
- out = {}
- d = Dataset.find params['id']
- rows = params[:per_page].to_i
- res = d.fetch_paginated_documents(params[:page].to_i, rows, params[:sort], params[:sort_order], params[:type])
- out[:documents] = render_to_string(layout: false,
- partial: "documents",
- locals: {docs: res[:docs], rows: rows, pagenum: params[:page].to_i})
- out[:pagination] = render_to_string(layout: false,
- partial: "pagination",
- locals: {nb_pages: res[:nb_pages].to_i, current_page: params[:page].to_i})
- render json: out
- end
-
- def list_datasets
- render json: current_user.datasets.to_json
- end
+ new_dataset.documents << { id: ca.id, type: "compound" }
+ else
+ new_dataset.documents << doc
+ end
+ end
+ begin
+ new_dataset.save!
+ render json: { status: "ok" }
+ rescue ActiveRecord::RecordNotUnique
+ render json: { status: "error", message: "A dataset with this title already exists." }
+ rescue ActiveRecord::RecordInvalid
+ render json: { status: "error", message: "The title should not be blank." }
+ end
+ end
+
+ ##
+ # Delete an existing dataset
+ def delete_dataset
+ dataset = Dataset.find(params[:dataset_id])
+ dataset_id = dataset.id
+ dataset.destroy
+ if session[:working_dataset] == dataset_id
+ if current_user.datasets.first
+ session[:working_dataset] = current_user.datasets.first.id
+ else
+ session[:working_dataset] = nil
+ end
+ end
+ end
+
+ ##
+ # Update the view of the list of datasets
+ def update_datasets_list
+ respond_to do |format|
+ format.js
+ end
+ end
+
+ ##
+ #
+ def set_working_dataset
+ session[:working_dataset] = params[:dataset_id]
+ @title = Dataset.find(session[:working_dataset]).title
+ puts "Setting working dataset to " + String(session[:working_dataset])
+ puts session.inspect
+ respond_to do |format|
+ format.js
+ end
+ end
+
+ def add_selected_documents
+ puts session.inspect
+ puts "User id from session is " + String(session[:current_user_id]) + " and working dataset is " + String(session[:working_dataset])
+ out = {}
+ @nb_added_docs = params[:documents_ids].size
+ dataset = Dataset.find(session[:working_dataset])
+ existing = dataset.add_documents params[:documents_ids] # Add docs and return existing ids
+ @nb_added_docs -= existing.size
+ title = dataset.title
+ message = " #{@nb_added_docs} document#{@nb_added_docs > 1 ? "s were" : " was"} added to your dataset.
"
+ message.concat "#{existing.size} document#{existing.size > 1 ? "s" : ""} already exist in this dataset.
" unless existing.empty?
+ # render partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
+ out["notif"] = render_to_string layout: false, partial: "shared/notification", locals: { notif_title: title, notif_content: message.html_safe }
+ out["nbissues"] = dataset.documents.select { |d| d["type"] == "issue" }.size
+ out["nbarticles"] = dataset.documents.select { |d| d["type"] == "article" }.size
+ out["nbdocs"] = out["nbissues"] + out["nbarticles"]
+ out["title"] = title
+ out["results_datasets"] = params[:documents_ids].map { |docid| [docid, render_to_string(layout: false, partial: "catalog/result_datasets", locals: { doc_id: docid })] }.to_h
+ render json: out
+ end
+
+ def add_compound
+ out = {}
+ dataset = Dataset.find(session[:working_dataset])
+ # dataset = Dataset.find(params[:working_dataset])
+ existing = dataset.add_compound params[:compound_id] # Add docs and return existing ids
+ title = dataset.title
+ message = " The compound article was added to your dataset.
"
+ out["notif"] = render_to_string layout: false, partial: "shared/notification", locals: { notif_title: title, notif_content: message.html_safe }
+ out["nbissues"] = dataset.documents.select { |d| d["type"] == "issue" }.size
+ out["nbarticles"] = dataset.documents.select { |d| d["type"] == "article" }.size
+ out["nbcompounds"] = dataset.documents.select { |d| d["type"] == "compound" }.size
+ out["nbdocs"] = out["nbissues"] + out["nbarticles"] + out["nbcompounds"]
+ out["title"] = title
+ render json: out
+ end
+
+ def remove_selected_documents
+ @nb_removed_docs = params[:documents_ids].size
+ puts "Remove selected documents called, session has working dataset with id " + String(session[:working_dataset])
+ dataset = Dataset.find(session[:working_dataset])
+ dataset.remove_documents params[:documents_ids]
+ redirect_to action: "show", id: dataset.id
+ end
+
+ def add_all_documents
+ SearchToDatasetWorker.perform_async(current_user.id, session[:working_dataset], params[:search_params].to_unsafe_h)
+ title = Dataset.find(session[:working_dataset]).title
+ message = "Documents are being added to your dataset. You will be notified when the operation is done.
"
+ render partial: "shared/notification", locals: { notif_title: title, notif_content: message.html_safe }
+ end
+
+ def export_dataset
+ ExportDatasetWorker.perform_async(current_user.id, params[:dataset_id], params[:export_type])
+ title = Dataset.find(params[:dataset_id]).title
+ message = "The export is being prepared. You will be notified when the operation is done.
"
+ render partial: "shared/notification", locals: { notif_title: title, notif_content: message.html_safe }
+ end
+
+ def toggle_sharing_status
+ @dataset = Dataset.find(params[:dataset_id])
+ @dataset.toggle!(:public)
+ render partial: "dataset_info"
+ end
+
+ def paginate
+ out = {}
+ d = Dataset.find params["id"]
+ rows = params[:per_page].to_i
+ res = d.fetch_paginated_documents(params[:page].to_i, rows, params[:sort], params[:sort_order], params[:type])
+ out[:documents] = render_to_string(layout: false,
+ partial: "documents",
+ locals: { docs: res[:docs], rows: rows, pagenum: params[:page].to_i })
+ out[:pagination] = render_to_string(layout: false,
+ partial: "pagination",
+ locals: { nb_pages: res[:nb_pages].to_i, current_page: params[:page].to_i })
+ render json: out
+ end
+
+ def list_datasets
+ render json: current_user.datasets.to_json
+ end
end
diff --git a/app/controllers/experiment_controller.rb b/app/controllers/experiment_controller.rb
index 5c89c48070c3dc380564f8752cb3399d970b4618..61d9bce3e41c0894c275c4b3bebd612b2fe9b1eb 100644
--- a/app/controllers/experiment_controller.rb
+++ b/app/controllers/experiment_controller.rb
@@ -1,118 +1,117 @@
class ExperimentController < ApplicationController
+ before_action :authenticate_user!
- before_action :authenticate_user!
+ def index
+ end
- def index
+ def create
+ experiment = Experiment.new
+ experiment.user = current_user
+ experiment.title = params[:title]
+ begin
+ experiment.save!
+ render json: { status: "ok" }
+ rescue ActiveRecord::RecordNotUnique
+ render json: { status: "error", message: "An experiment with this title already exists." }
+ rescue ActiveRecord::RecordInvalid
+ render json: { status: "error", message: "The title should not be blank." }
end
+ end
- def create
- experiment = Experiment.new
- experiment.user = current_user
- experiment.title = params[:title]
- begin
- experiment.save!
- render json: {status: 'ok'}
- rescue ActiveRecord::RecordNotUnique
- render json: {status: "error", message: "An experiment with this title already exists."}
- rescue ActiveRecord::RecordInvalid
- render json: {status: "error", message: "The title should not be blank."}
- end
+ def delete
+ experiment = Experiment.find(params[:experiment_id])
+ root_ids = experiment.description["children"].map { |root| root["tool"]["id"] }
+ root_ids.each do |root_id|
+ Tool.destroy(experiment.delete_tool(root_id))
end
+ experiment.destroy
+ end
- def delete
- experiment = Experiment.find(params[:experiment_id])
- root_ids = experiment.description["children"].map{|root| root['tool']['id'] }
- root_ids.each do |root_id|
- Tool.destroy(experiment.delete_tool(root_id))
- end
- experiment.destroy
- end
-
- def show
- @experiment = Experiment.find params[:id]
- @tools = @experiment.load_tools
- @tools = JSON.parse(File.read("#{Rails.root}/lib/newspapers_tools.json"))
- @tools['tools']['processors'].delete_if{ |h| h["type"] == "splitter" }
- end
+ def show
+ @experiment = Experiment.find params[:id]
+ @tools = @experiment.load_tools
+ @tools = JSON.parse(File.read("#{Rails.root}/lib/newspapers_tools.json"))
+ @tools["tools"]["processors"].delete_if { |h| h["type"] == "splitter" }
+ end
- def update_experiments_list
- respond_to do |format|
- format.js
- end
+ def update_experiments_list
+ respond_to do |format|
+ format.js
end
+ end
- def add_tool
- @experiment = Experiment.find(params[:id])
- tool_params = JSON.parse params[:tool]
- tool = Tool.new
- tool.tool_type = tool_params['type']
- tool.input_type = tool_params['input_type']
- tool.output_type = tool_params['output_type']
- tool.parameters = tool_params['parameters']
- tool.status = "created"
- tool.parent_id = params[:parent_id]#(params[:parent_id] == "") ? nil : Tool.find(params[:parent_id])
- tool.experiment = @experiment
- tool.save!
- @experiment.add_tool(params[:parent_id].to_i, tool)
- @experiment.save!
- render 'experiment/update_experiment_area'
- end
+ def add_tool
+ @experiment = Experiment.find(params[:id])
+ tool_params = JSON.parse params[:tool]
+ tool = Tool.new
+ tool.tool_type = tool_params["type"]
+ tool.input_type = tool_params["input_type"]
+ tool.output_type = tool_params["output_type"]
+ tool.parameters = tool_params["parameters"]
+ tool.status = "created"
+ tool.parent_id = params[:parent_id] #(params[:parent_id] == "") ? nil : Tool.find(params[:parent_id])
+ tool.experiment = @experiment
+ tool.save!
+ @experiment.add_tool(params[:parent_id].to_i, tool)
+ @experiment.save!
+ render "experiment/update_experiment_area"
+ end
- def delete_tool
- @experiment = Experiment.find(params[:id])
- tools_to_destroy_ids = @experiment.delete_tool(params[:tool_id].to_i)
- @experiment.save!
- Tool.destroy(tools_to_destroy_ids)
- render 'experiment/update_experiment_area'
- end
+ def delete_tool
+ @experiment = Experiment.find(params[:id])
+ tools_to_destroy_ids = @experiment.delete_tool(params[:tool_id].to_i)
+ @experiment.save!
+ Tool.destroy(tools_to_destroy_ids)
+ render "experiment/update_experiment_area"
+ end
- def edit_tool_form
- @tool = Tool.find(params[:tool_id])
- render partial: 'tool/parameters', locals: {tool: @tool}
- end
+ def edit_tool_form
+ @tool = Tool.find(params[:tool_id])
+ render partial: "tool/parameters", locals: { tool: @tool }
+ end
- def edit_tool
- @experiment = Experiment.find(params[:id])
- @tool = Tool.find(params[:tool_id])
- modified = false
- @tool.parameters.map! do |param|
- if param['value'] != params[:parameters][param['name']]
- modified = true
- end
- param['value'] = params[:parameters][param['name']]
- param
- end
- @tool.status = "configured" if modified
- @tool.save!
- render 'experiment/update_experiment_area'
+ def edit_tool
+ @experiment = Experiment.find(params[:id])
+ @tool = Tool.find(params[:tool_id])
+ modified = false
+ @tool.parameters.map! do |param|
+ if param["value"] != params[:parameters][param["name"]]
+ modified = true
+ end
+ param["value"] = params[:parameters][param["name"]]
+ param
end
+ @tool.status = "configured" if modified
+ @tool.save!
+ render "experiment/update_experiment_area"
+ end
- def tool_results
- @experiment = Experiment.find(params[:id])
- @tool = Tool.find(params[:tool_id])
- render partial: 'tool/results', locals: {tool: @tool, experiment: @experiment}
- end
+ def tool_results
+ @experiment = Experiment.find(params[:id])
+ @tool = Tool.find(params[:tool_id])
+ render partial: "tool/results", locals: { tool: @tool, experiment: @experiment }
+ end
- def run_tool
- @experiment = Experiment.find(params[:id])
- @tool = Tool.find(params[:tool_id])
- @tool.run()
- render 'experiment/update_experiment_area'
- end
+ def run_tool
+ @experiment = Experiment.find(params[:id])
+ @tool = Tool.find(params[:tool_id])
+ @tool.run()
+ render "experiment/update_experiment_area"
+ end
- def run_experiment
- out = {}
- @experiment = Experiment.find(params[:experiment_id])
- ids = @experiment.get_tool_ids
- running = false
- ids.map{|id| Tool.find(id)}.each do |tool|
- if tool.runnable?
- tool.run(true)
- running = true
- end
- end
- out[:html_tree] = render_to_string partial: "tree", locals: {experiment: @experiment}
- out[:experiment_running] = running
- render json: out
+ def run_experiment
+ out = {}
+ @experiment = Experiment.find(params[:experiment_id])
+ ids = @experiment.get_tool_ids
+ running = false
+ ids.map { |id| Tool.find(id) }.each do |tool|
+ if tool.runnable?
+ tool.run(true)
+ running = true
+ end
end
+ out[:html_tree] = render_to_string partial: "tree", locals: { experiment: @experiment }
+ out[:experiment_running] = running
+ render json: out
+ end
end
diff --git a/app/controllers/notification_controller.rb b/app/controllers/notification_controller.rb
index 53306f026c34a52b604737965d3f45069ccf4ab3..43e7a448386440766ba963f829149080605773cf 100644
--- a/app/controllers/notification_controller.rb
+++ b/app/controllers/notification_controller.rb
@@ -1,3 +1,3 @@
class NotificationController < ApplicationController
-
-end
\ No newline at end of file
+ before_action :authenticate_user!
+end
diff --git a/app/controllers/tool_controller.rb b/app/controllers/tool_controller.rb
index d9fbf32fe8fd70f9fdd4e0191763b1bc5e9a62ff..2e047a96589e49ddef709a411faa2b229222ecab 100644
--- a/app/controllers/tool_controller.rb
+++ b/app/controllers/tool_controller.rb
@@ -1,26 +1,21 @@
class ToolController < ApplicationController
+ before_action :authenticate_user!
- before_action :authenticate_user!
+ def show
+ end
- def show
+ def create
+ end
- end
+ def update
+ end
- def create
+ def destroy
+ end
- end
+ private
- def update
-
- end
-
- def destroy
-
- end
-
- private
-
- def tool_params
- params.require(:tool).permit(:parameters, :results, :status)
- end
+ def tool_params
+ params.require(:tool).permit(:parameters, :results, :status)
+ end
end
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index a69ed066bcadd2cfc54e470d06f04d076e83561c..15f2511626a73c1182080ec88be05629588196fc 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -1,7 +1,5 @@
module ApplicationHelper
-
- def set_page_title(title)
- content_for :page_title, title
- end
-
+ def set_page_title(title)
+ content_for :page_title, title
+ end
end
diff --git a/app/helpers/experiment_helper.rb b/app/helpers/experiment_helper.rb
index f03f142c96df04bd436fb1db1f7329e956f801bd..9bc2e4cf8fe718cfc7a8567a48f6f3bba357c6e5 100644
--- a/app/helpers/experiment_helper.rb
+++ b/app/helpers/experiment_helper.rb
@@ -1,17 +1,15 @@
module ExperimentHelper
-
- def recursive_display(tree, tools)
- if tree.has_key? "tool"
- concat "".html_safe
- concat render partial: 'tool/canvas_tool', locals: {tool: tools[tree['tool']['id']]}
- concat "".html_safe
- end
- tree['children'].each do |node|
- recursive_display(node, tools)
- end
- concat '
".html_safe
- concat "".html_safe if tree.has_key? "tool"
+ def recursive_display(tree, tools)
+ if tree.has_key? "tool"
+ concat "".html_safe
+ concat render partial: "tool/canvas_tool", locals: { tool: tools[tree["tool"]["id"]] }
+ concat "".html_safe
end
-
-end
\ No newline at end of file
+ tree["children"].each do |node|
+ recursive_display(node, tools)
+ end
+ concat '
".html_safe
+ concat "".html_safe if tree.has_key? "tool"
+ end
+end
diff --git a/app/helpers/named_entities_helper.rb b/app/helpers/named_entities_helper.rb
index 041962362c6aafc418b8e93d6ee4b621aa95267b..e9c2536ec0acb2ac3a1da76a767c531ab6b5d30c 100644
--- a/app/helpers/named_entities_helper.rb
+++ b/app/helpers/named_entities_helper.rb
@@ -1,40 +1,38 @@
module NamedEntitiesHelper
-
- def get_linked_entities entities
- priority_language = [I18n.locale, 'en', 'de', 'fr', 'fi', 'sv']
- ids = entities.select{ |label| label != "" && label != nil }
- return {} if ids.empty?
- out = {}
- SolrSearcher.query({q: "*:*", fq: "id:(#{ids.join(' ')})", fl: "*", rows: 99999})['response']['docs'].map do |res|
- priority_language.each do |lang|
- unless res["label_#{lang}_ssi"].nil?
- out[res['id']] = {kb_url: res['kb_url_ssi'], label: res["label_#{lang}_ssi"]}
- break
- end
- end
+ def get_linked_entities(entities)
+ priority_language = [I18n.locale, "en", "de", "fr", "fi", "sv"]
+ ids = entities.select { |label| label != "" && label != nil }
+ return {} if ids.empty?
+ out = {}
+ SolrSearcher.query({ q: "*:*", fq: "id:(#{ids.join(" ")})", fl: "*", rows: 99999 })["response"]["docs"].map do |res|
+ priority_language.each do |lang|
+ unless res["label_#{lang}_ssi"].nil?
+ out[res["id"]] = { kb_url: res["kb_url_ssi"], label: res["label_#{lang}_ssi"] }
+ break
end
- out
+ end
end
+ out
+ end
- def get_entity_label(options={})
- priority_language = [I18n.locale, 'en', 'de', 'fr', 'fi', 'sv']
- if options.class == Array
- out = {}
- unless options.empty?
- docs = SolrSearcher.query({q: "*:*", fq: "id:(#{options.join(' ')})", fl: "*", rows: 99999})['response']['docs']
- docs.map do |doc|
- priority_language.each do |lang|
- unless doc["label_#{lang}_ssi"].nil?
- out[doc['id']] = doc["label_#{lang}_ssi"]
- break
- end
- end
- end
+ def get_entity_label(options = {})
+ priority_language = [I18n.locale, "en", "de", "fr", "fi", "sv"]
+ if options.class == Array
+ out = {}
+ unless options.empty?
+ docs = SolrSearcher.query({ q: "*:*", fq: "id:(#{options.join(" ")})", fl: "*", rows: 99999 })["response"]["docs"]
+ docs.map do |doc|
+ priority_language.each do |lang|
+ unless doc["label_#{lang}_ssi"].nil?
+ out[doc["id"]] = doc["label_#{lang}_ssi"]
+ break
end
- return out
- else
- @entities_labels[options] # set in catalog_controller#index
+ end
end
+ end
+ return out
+ else
+ @entities_labels[options] # set in catalog_controller#index
end
-
-end
\ No newline at end of file
+ end
+end
diff --git a/app/helpers/search_helper.rb b/app/helpers/search_helper.rb
index d7405707c1778316de129fdba7fbfef3366b83d4..e9eb2acf15429dc437fc7427d3f7acad673c105b 100644
--- a/app/helpers/search_helper.rb
+++ b/app/helpers/search_helper.rb
@@ -1,37 +1,35 @@
module SearchHelper
+ def current_page_params
+ params.to_unsafe_h.slice("q", "page", "per_page", "sort", "search_type", "f")
+ end
- def current_page_params
- params.to_unsafe_h.slice('q', 'page', 'per_page','sort', 'search_type', 'f')
+ def merge_facets(parameters, new)
+ parameters.merge(new) do |key, oldval, newval|
+ oldval.merge(newval)
end
+ end
- def merge_facets(parameters, new)
- parameters.merge(new) do |key, oldval, newval|
- oldval.merge(newval)
- end
- end
-
- def convert_solr_date_to_datepicker_date solr_date
- DateTime.parse(solr_date).strftime("%Y-%m-%d")
- end
+ def convert_solr_date_to_datepicker_date(solr_date)
+ DateTime.parse(solr_date).strftime("%Y-%m-%d")
+ end
- def convert_datepicker_date_to_solr_date solr_date
- DateTime.parse(solr_date).strftime("%Y-%m-%d")
- end
+ def convert_datepicker_date_to_solr_date(solr_date)
+ DateTime.parse(solr_date).strftime("%Y-%m-%d")
+ end
- def search_constraints
- constraints = []
- if current_page_params[:f]
- current_page_params[:f].each do |f, vals|
- if f == "date_created_dtsi"
- constraints << {label: f, value: "From #{vals['from']} To #{vals['to']}"}
- else
- vals.each do |val|
- constraints << {label: f, value: val}
- end
- end
- end
+ def search_constraints
+ constraints = []
+ if current_page_params[:f]
+ current_page_params[:f].each do |f, vals|
+ if f == "date_created_dtsi"
+ constraints << { label: f, value: "From #{vals["from"]} To #{vals["to"]}" }
+ else
+ vals.each do |val|
+ constraints << { label: f, value: val }
+ end
end
- constraints
+ end
end
-
-end
\ No newline at end of file
+ constraints
+ end
+end
diff --git a/app/mailers/application_mailer.rb b/app/mailers/application_mailer.rb
index 286b2239d139960190594225e0134fe1a5c05370..3c34c8148f105d699e8ec9b769531192f8637d9a 100644
--- a/app/mailers/application_mailer.rb
+++ b/app/mailers/application_mailer.rb
@@ -1,4 +1,4 @@
class ApplicationMailer < ActionMailer::Base
- default from: 'from@example.com'
- layout 'mailer'
+ default from: "from@example.com"
+ layout "mailer"
end
diff --git a/app/models/article.rb b/app/models/article.rb
index 145c405ad2c50adf6a5cd1884ed038d708c07bb4..27451ce18cc0780e43af3d90f8308c6713ac85f6 100644
--- a/app/models/article.rb
+++ b/app/models/article.rb
@@ -1,100 +1,99 @@
class Article
+ attr_accessor :id, :title, :all_text, :date_created, :language, :canvases_parts, :newspaper, :issue_id, :thumbnail_url, :bbox
- attr_accessor :id, :title, :all_text, :date_created, :language, :canvases_parts, :newspaper, :issue_id, :thumbnail_url, :bbox
+ def self.from_solr(id)
+ solr_doc = SolrSearcher.get_doc_by_id id
+ Article.from_solr_doc solr_doc
+ end
- def self.from_solr id
- solr_doc = SolrSearcher.get_doc_by_id id
- Article.from_solr_doc solr_doc
- end
+ def self.from_solr_doc(solr_doc)
+ a = Article.new
+ a.id = solr_doc["id"]
+ a.title = solr_doc["title_ssi"]
+ a.language = solr_doc["language_ssi"]
+ a.all_text = solr_doc["all_text_t#{a.language}_siv"]
+ a.date_created = solr_doc["date_created_ssi"]
+ a.issue_id = solr_doc["from_issue_ssi"]
+ a.newspaper = solr_doc["member_of_collection_ids_ssim"].first
+ a.thumbnail_url = solr_doc["thumbnail_url_ss"]
+ a.canvases_parts = solr_doc["canvases_parts_ssm"]
+ a.bbox = a.get_location
+ a
+ end
- def self.from_solr_doc solr_doc
- a = Article.new
- a.id = solr_doc['id']
- a.title = solr_doc['title_ssi']
- a.language = solr_doc['language_ssi']
- a.all_text = solr_doc["all_text_t#{a.language}_siv"]
- a.date_created = solr_doc['date_created_ssi']
- a.issue_id = solr_doc['from_issue_ssi']
- a.newspaper = solr_doc['member_of_collection_ids_ssim'].first
- a.thumbnail_url = solr_doc['thumbnail_url_ss']
- a.canvases_parts = solr_doc['canvases_parts_ssm']
- a.bbox = a.get_location
- a
- end
+ def to_solr(page_iiif_url)
+ solr_doc = {}
+ solr_doc["id"] = self.id
+ solr_doc["title_ssi"] = self.title
+ solr_doc["language_ssi"] = self.language
+ solr_doc["all_text_t#{self.language}_siv"] = self.all_text
+ solr_doc["all_text_unstemmed_t#{self.language}_siv"] = self.all_text
+ solr_doc["date_created_ssi"] = self.date_created
+ solr_doc["date_created_dtsi"] = DateTime.parse(self.date_created).strftime("%Y-%m-%dT%H:%M:%SZ")
+ solr_doc["year_isi"] = solr_doc["date_created_ssi"][0..3].to_i
+ d = DateTime.parse solr_doc["date_created_dtsi"]
+ solr_doc["month_isi"] = d.month
+ solr_doc["day_isi"] = d.wday
+ solr_doc["from_issue_ssi"] = self.issue_id
+ solr_doc["member_of_collection_ids_ssim"] = self.newspaper
+ solr_doc["canvases_parts_ssm"] = self.canvases_parts
+ solr_doc["thumbnail_url_ss"] = self.get_iiif_url(page_iiif_url)
+ solr_doc["has_model_ssim"] = "Article"
+ solr_doc
+ end
- def to_solr(page_iiif_url)
- solr_doc = {}
- solr_doc['id'] = self.id
- solr_doc['title_ssi'] = self.title
- solr_doc["language_ssi"] = self.language
- solr_doc["all_text_t#{self.language}_siv"] = self.all_text
- solr_doc["all_text_unstemmed_t#{self.language}_siv"] = self.all_text
- solr_doc['date_created_ssi'] = self.date_created
- solr_doc['date_created_dtsi'] = DateTime.parse(self.date_created).strftime('%Y-%m-%dT%H:%M:%SZ')
- solr_doc['year_isi'] = solr_doc['date_created_ssi'][0..3].to_i
- d = DateTime.parse solr_doc["date_created_dtsi"]
- solr_doc['month_isi'] = d.month
- solr_doc['day_isi'] = d.wday
- solr_doc['from_issue_ssi'] = self.issue_id
- solr_doc['member_of_collection_ids_ssim'] = self.newspaper
- solr_doc['canvases_parts_ssm'] = self.canvases_parts
- solr_doc['thumbnail_url_ss'] = self.get_iiif_url(page_iiif_url)
- solr_doc['has_model_ssim'] = 'Article'
- solr_doc
+ def get_thumbnail
+ if Rails.configuration.iiif_sources[:local].include? self.newspaper
+ pagenum = self.canvases_parts[0][self.canvases_parts[0].rindex("_") + 1...self.canvases_parts[0].rindex("#")].to_i
+ self.get_iiif_url("https://iiif.newseye.eu/iiif/#{self.newspaper}/#{self.issue_id}_page_#{pagenum}.ptif")
+ elsif Rails.configuration.iiif_sources[:external].include? self.newspaper
+ self.thumbnail_url
+ elsif Rails.configuration.iiif_sources[:external_onb].include? self.newspaper
+ self.thumbnail_url
end
+ end
- def get_thumbnail
- if Rails.configuration.iiif_sources[:local].include? self.newspaper
- pagenum = self.canvases_parts[0][self.canvases_parts[0].rindex('_')+1...self.canvases_parts[0].rindex('#')].to_i
- self.get_iiif_url("https://iiif.newseye.eu/iiif/#{self.newspaper}/#{self.issue_id}_page_#{pagenum}.ptif")
- elsif Rails.configuration.iiif_sources[:external].include? self.newspaper
- self.thumbnail_url
- elsif Rails.configuration.iiif_sources[:external_onb].include? self.newspaper
- self.thumbnail_url
- end
- end
+ def get_location
+ coords = self.canvases_parts.map { |c| c[c.rindex("#xywh=") + 6..-1].split(",").map(&:to_i) }
+ min_x = coords.map { |coord| coord[0] }.min
+ max_x = coords.map { |coord| coord[0] + coord[2] }.max
+ min_y = coords.map { |coord| coord[1] }.min
+ max_y = coords.map { |coord| coord[1] + coord[3] }.max
+ canvas_coords = [min_x, max_x, min_y, max_y]
+ canvas_size = [canvas_coords[1] - canvas_coords[0], canvas_coords[3] - canvas_coords[2]]
+ [min_x, min_y, canvas_size[0], canvas_size[1]]
+ end
- def get_location
- coords = self.canvases_parts.map { |c| c[c.rindex('#xywh=')+6..-1].split(',').map(&:to_i) }
- min_x = coords.map{ |coord| coord[0] }.min
- max_x = coords.map{ |coord| coord[0] + coord[2] }.max
- min_y = coords.map{ |coord| coord[1] }.min
- max_y = coords.map{ |coord| coord[1] + coord[3] }.max
- canvas_coords = [min_x, max_x, min_y, max_y]
- canvas_size = [canvas_coords[1]-canvas_coords[0], canvas_coords[3]-canvas_coords[2]]
- [min_x,min_y,canvas_size[0],canvas_size[1]]
- end
+ def get_iiif_url(page_iiif_url)
+ canvas_url = self.canvases_parts[0]
+ coords = self.canvases_parts.map { |c| c[c.rindex("#xywh=") + 6..-1].split(",").map(&:to_i) }
+ min_x = coords.map { |coord| coord[0] }.min
+ max_x = coords.map { |coord| coord[0] + coord[2] }.max
+ min_y = coords.map { |coord| coord[1] }.min
+ max_y = coords.map { |coord| coord[1] + coord[3] }.max
+ pagenum = canvas_url[canvas_url.rindex("_") + 1...canvas_url.rindex("#")].to_i
+ "#{page_iiif_url}/#{min_x},#{min_y},#{max_x - min_x},#{max_y - min_y}/full/0/default.jpg"
+ end
- def get_iiif_url(page_iiif_url)
- canvas_url = self.canvases_parts[0]
- coords = self.canvases_parts.map { |c| c[c.rindex('#xywh=')+6..-1].split(',').map(&:to_i) }
- min_x = coords.map{ |coord| coord[0] }.min
- max_x = coords.map{ |coord| coord[0] + coord[2] }.max
- min_y = coords.map{ |coord| coord[1] }.min
- max_y = coords.map{ |coord| coord[1] + coord[3] }.max
- pagenum = canvas_url[canvas_url.rindex('_')+1...canvas_url.rindex('#')].to_i
- "#{page_iiif_url}/#{min_x},#{min_y},#{max_x-min_x},#{max_y-min_y}/full/0/default.jpg"
+ def self.named_entities(article_id)
+ nems = SolrSearcher.query({ q: "article_id_ssi:#{article_id}", rows: 1000000 })["response"]["docs"]
+ output = { LOC: {}, PER: {}, ORG: {}, HumanProd: {} }
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "LOC" }.each do |ne_solr|
+ output[:LOC][ne_solr["linked_entity_ssi"]] = [] unless output[:LOC].has_key? ne_solr["linked_entity_ssi"]
+ output[:LOC][ne_solr["linked_entity_ssi"]].append(ne_solr)
end
-
- def self.named_entities(article_id)
- nems = SolrSearcher.query({q:"article_id_ssi:#{article_id}", rows: 1000000})['response']['docs']
- output = {LOC: {}, PER: {}, ORG: {}, HumanProd: {}}
- nems.select {|ne_solr| ne_solr['type_ssi'] == "LOC"}.each do |ne_solr|
- output[:LOC][ne_solr['linked_entity_ssi']] = [] unless output[:LOC].has_key? ne_solr['linked_entity_ssi']
- output[:LOC][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- nems.select {|ne_solr| ne_solr['type_ssi'] == "PER"}.each do |ne_solr|
- output[:PER][ne_solr['linked_entity_ssi']] = [] unless output[:PER].has_key? ne_solr['linked_entity_ssi']
- output[:PER][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- nems.select {|ne_solr| ne_solr['type_ssi'] == "ORG"}.each do |ne_solr|
- output[:ORG][ne_solr['linked_entity_ssi']] = [] unless output[:ORG].has_key? ne_solr['linked_entity_ssi']
- output[:ORG][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- nems.select {|ne_solr| ne_solr['type_ssi'] == "HumanProd"}.each do |ne_solr|
- output[:HumanProd][ne_solr['linked_entity_ssi']] = [] unless output[:HumanProd].has_key? ne_solr['linked_entity_ssi']
- output[:HumanProd][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- output
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "PER" }.each do |ne_solr|
+ output[:PER][ne_solr["linked_entity_ssi"]] = [] unless output[:PER].has_key? ne_solr["linked_entity_ssi"]
+ output[:PER][ne_solr["linked_entity_ssi"]].append(ne_solr)
+ end
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "ORG" }.each do |ne_solr|
+ output[:ORG][ne_solr["linked_entity_ssi"]] = [] unless output[:ORG].has_key? ne_solr["linked_entity_ssi"]
+ output[:ORG][ne_solr["linked_entity_ssi"]].append(ne_solr)
+ end
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "HumanProd" }.each do |ne_solr|
+ output[:HumanProd][ne_solr["linked_entity_ssi"]] = [] unless output[:HumanProd].has_key? ne_solr["linked_entity_ssi"]
+ output[:HumanProd][ne_solr["linked_entity_ssi"]].append(ne_solr)
end
-end
\ No newline at end of file
+ output
+ end
+end
diff --git a/app/models/compound_article.rb b/app/models/compound_article.rb
index 5372b31deaffddfdd73d0f4fa3295f9391103214..dec031193b74e0b13282e1a347a490d0fcbcb0da 100644
--- a/app/models/compound_article.rb
+++ b/app/models/compound_article.rb
@@ -1,6 +1,4 @@
class CompoundArticle < ActiveRecord::Base
-
- belongs_to :user, optional: false
- validates :title, length: { minimum: 1 }
-
-end
\ No newline at end of file
+ belongs_to :user, optional: false
+ validates :title, length: { minimum: 1 }
+end
diff --git a/app/models/concerns/abstract_searcher.rb b/app/models/concerns/abstract_searcher.rb
index 0bced801a3d7e2c1483a9f0165a9a8678d4e10c1..d569b1d1c7890fdd6a71baacfc3be5870a0170bb 100644
--- a/app/models/concerns/abstract_searcher.rb
+++ b/app/models/concerns/abstract_searcher.rb
@@ -1,12 +1,11 @@
module AbstractSearcher
- extend ActiveSupport::Concern
+ extend ActiveSupport::Concern
- def self.query
- raise NotImplementedError, "Subclasses must define `query`."
- end
+ def self.query
+ raise NotImplementedError, "Subclasses must define `query`."
+ end
- def self.get_doc_by_id(id)
- raise NotImplementedError, "Subclasses must define `get_doc_by_id`."
- end
-
-end
\ No newline at end of file
+ def self.get_doc_by_id(id)
+ raise NotImplementedError, "Subclasses must define `get_doc_by_id`."
+ end
+end
diff --git a/app/models/current.rb b/app/models/current.rb
index 12113733160e8569eaccbe80847d987b56803b30..73a9744b335eab8688cffa0c1207a6248c08036d 100644
--- a/app/models/current.rb
+++ b/app/models/current.rb
@@ -1,3 +1,3 @@
class Current < ActiveSupport::CurrentAttributes
- attribute :user
+ attribute :user
end
diff --git a/app/models/dataset.rb b/app/models/dataset.rb
index 59b49ae3ac77cfd28bf941640a14451f3a133a64..be88129d01b61644d3a95b36df75de93e92b505e 100644
--- a/app/models/dataset.rb
+++ b/app/models/dataset.rb
@@ -1,122 +1,121 @@
class Dataset < ActiveRecord::Base
- # after_find :nb_issues, :nb_articles
- belongs_to :user, optional: false
- validates :title, length: { minimum: 1 }
+ # after_find :nb_issues, :nb_articles
+ belongs_to :user, optional: false
+ validates :title, length: { minimum: 1 }
- def add_documents(documents_ids)
- existing = []
- documents_ids.each do |doc_id|
- if self.documents.any?{ |doc| doc['id'] == doc_id }
- existing << doc_id
- else
- doc_type = doc_id.index("_article_").nil? ? "issue" : "article"
- self.documents << {id: doc_id, type: doc_type}
- end
- end
- self.save
- return existing
- end
-
- def add_compound(compound_id)
- existing = []
- if self.documents.any?{ |doc| doc['id'] == compound_id }
- existing << compound_id
- else
- doc_type = "compound"
- self.documents << {id: compound_id, type: doc_type}
- end
- self.save
- return existing
+ def add_documents(documents_ids)
+ existing = []
+ documents_ids.each do |doc_id|
+ if self.documents.any? { |doc| doc["id"] == doc_id }
+ existing << doc_id
+ else
+ doc_type = doc_id.index("_article_").nil? ? "issue" : "article"
+ self.documents << { id: doc_id, type: doc_type }
+ end
end
+ self.save
+ return existing
+ end
- def remove_documents(documents_ids)
- self.documents.delete_if{ |elt| documents_ids.include? elt['id'] }
- self.save
+ def add_compound(compound_id)
+ existing = []
+ if self.documents.any? { |doc| doc["id"] == compound_id }
+ existing << compound_id
+ else
+ doc_type = "compound"
+ self.documents << { id: compound_id, type: doc_type }
end
+ self.save
+ return existing
+ end
- def contains doc_id
- self.documents.index { |doc| doc['id'] == doc_id }.nil? ? false : true
- end
+ def remove_documents(documents_ids)
+ self.documents.delete_if { |elt| documents_ids.include? elt["id"] }
+ self.save
+ end
- def nb_issues
- self.documents.select do |doc|
- doc['type'] == 'issue'
- end.size
- end
+ def contains(doc_id)
+ self.documents.index { |doc| doc["id"] == doc_id }.nil? ? false : true
+ end
- def nb_articles
- self.documents.select do |doc|
- doc['type'] == 'article'
- end.size
- end
+ def nb_issues
+ self.documents.select do |doc|
+ doc["type"] == "issue"
+ end.size
+ end
- def nb_compound_articles
- self.documents.select do |doc|
- doc['type'] == 'compound'
- end.size
- end
+ def nb_articles
+ self.documents.select do |doc|
+ doc["type"] == "article"
+ end.size
+ end
- def fetch_paginated_documents(page, per_page, sort, sort_order, type, recursive=false)
- docs = self.documents.select {|doc| type == "all" || doc['type'] == type }
+ def nb_compound_articles
+ self.documents.select do |doc|
+ doc["type"] == "compound"
+ end.size
+ end
- nb_pages = (docs.size / per_page.to_f).ceil
- nb_pages = 1 if nb_pages == 0
- sort = (sort == "default") ? "score" : sort
- solr_docs = nil
+ def fetch_paginated_documents(page, per_page, sort, sort_order, type, recursive = false)
+ docs = self.documents.select { |doc| type == "all" || doc["type"] == type }
- compounds_ids = docs.select{|d| d['type'] == "compound" }.map{ |d| d['id'] }
- compound_articles = CompoundArticle.find(compounds_ids)
+ nb_pages = (docs.size / per_page.to_f).ceil
+ nb_pages = 1 if nb_pages == 0
+ sort = (sort == "default") ? "score" : sort
+ solr_docs = nil
+ compounds_ids = docs.select { |d| d["type"] == "compound" }.map { |d| d["id"] }
+ compound_articles = CompoundArticle.find(compounds_ids)
- solr_ids = docs.select{|d| d['type'] != "compound" }.map{ |d| d['id'] }
- unless solr_ids.empty?
- solr_docs = SolrSearcher.query({
+ solr_ids = docs.select { |d| d["type"] != "compound" }.map { |d| d["id"] }
+ unless solr_ids.empty?
+ solr_docs = SolrSearcher.query({
q: "*:*",
- fq: "id:(#{solr_ids.join(' ')})",
+ fq: "id:(#{solr_ids.join(" ")})",
rows: per_page,
sort: "#{sort} #{sort_order}",
- start: (page-1)*per_page
- })['response']['docs']
- solr_docs.map! do |solr_doc|
- if solr_doc['id'].index("_article_").nil?
- Issue.from_solr_doc solr_doc
- else
- Article.from_solr_doc solr_doc
- end
- end
- end
- if recursive and page < nb_pages and !solr_docs.nil?
- solr_docs = solr_docs.concat fetch_paginated_documents(page+1, per_page, sort, sort_order, type, true)[:docs]
+ start: (page - 1) * per_page,
+ })["response"]["docs"]
+ solr_docs.map! do |solr_doc|
+ if solr_doc["id"].index("_article_").nil?
+ Issue.from_solr_doc solr_doc
+ else
+ Article.from_solr_doc solr_doc
end
- return {docs: solr_docs.nil? ? compound_articles : solr_docs+compound_articles, nb_pages: nb_pages}
+ end
+ end
+ if recursive and page < nb_pages and !solr_docs.nil?
+ solr_docs = solr_docs.concat fetch_paginated_documents(page + 1, per_page, sort, sort_order, type, true)[:docs]
end
+ return { docs: solr_docs.nil? ? compound_articles : solr_docs + compound_articles, nb_pages: nb_pages }
+ end
- def named_entities
- article_ids = self.documents.select {|d| d['type'] == 'article' }.map{|d| d['id']}
- issue_ids = self.documents.select {|d| d['type'] == 'issue' }.map{|d| d['id']}
- parts_ids = self.documents.select {|d| d['type'] == 'compound' }.map{|d| CompoundArticle.find(d['id']).parts}.flatten.uniq
- nems = []
- nems = SolrSearcher.query({q: "*:*", fq: "article_id_ssi:(#{article_ids.join(' OR ')})", rows: 1000000})['response']['docs'] unless article_ids.empty?
- nems += SolrSearcher.query({q: "*:*", fq: "article_id_ssi:(#{parts_ids.join(' OR ')})", rows: 1000000})['response']['docs'] unless parts_ids.empty?
- nems += SolrSearcher.query({q: "*:*", fq: "issue_id_ssi:(#{issue_ids.join(' OR ')})", rows: 1000000})['response']['docs'] unless issue_ids.empty?
- output = {LOC: {}, PER: {}, ORG: {}, HumanProd: {}}
- nems.select {|ne_solr| ne_solr['type_ssi'] == "LOC"}.each do |ne_solr|
- output[:LOC][ne_solr['linked_entity_ssi']] = [] unless output[:LOC].has_key? ne_solr['linked_entity_ssi']
- output[:LOC][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- nems.select {|ne_solr| ne_solr['type_ssi'] == "PER"}.each do |ne_solr|
- output[:PER][ne_solr['linked_entity_ssi']] = [] unless output[:PER].has_key? ne_solr['linked_entity_ssi']
- output[:PER][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- nems.select {|ne_solr| ne_solr['type_ssi'] == "ORG"}.each do |ne_solr|
- output[:ORG][ne_solr['linked_entity_ssi']] = [] unless output[:ORG].has_key? ne_solr['linked_entity_ssi']
- output[:ORG][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- nems.select {|ne_solr| ne_solr['type_ssi'] == "HumanProd"}.each do |ne_solr|
- output[:HumanProd][ne_solr['linked_entity_ssi']] = [] unless output[:HumanProd].has_key? ne_solr['linked_entity_ssi']
- output[:HumanProd][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- output
+ def named_entities
+ article_ids = self.documents.select { |d| d["type"] == "article" }.map { |d| d["id"] }
+ issue_ids = self.documents.select { |d| d["type"] == "issue" }.map { |d| d["id"] }
+ parts_ids = self.documents.select { |d| d["type"] == "compound" }.map { |d| CompoundArticle.find(d["id"]).parts }.flatten.uniq
+ nems = []
+ nems = SolrSearcher.query({ q: "*:*", fq: "article_id_ssi:(#{article_ids.join(" OR ")})", rows: 1000000 })["response"]["docs"] unless article_ids.empty?
+ nems += SolrSearcher.query({ q: "*:*", fq: "article_id_ssi:(#{parts_ids.join(" OR ")})", rows: 1000000 })["response"]["docs"] unless parts_ids.empty?
+ nems += SolrSearcher.query({ q: "*:*", fq: "issue_id_ssi:(#{issue_ids.join(" OR ")})", rows: 1000000 })["response"]["docs"] unless issue_ids.empty?
+ output = { LOC: {}, PER: {}, ORG: {}, HumanProd: {} }
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "LOC" }.each do |ne_solr|
+ output[:LOC][ne_solr["linked_entity_ssi"]] = [] unless output[:LOC].has_key? ne_solr["linked_entity_ssi"]
+ output[:LOC][ne_solr["linked_entity_ssi"]].append(ne_solr)
+ end
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "PER" }.each do |ne_solr|
+ output[:PER][ne_solr["linked_entity_ssi"]] = [] unless output[:PER].has_key? ne_solr["linked_entity_ssi"]
+ output[:PER][ne_solr["linked_entity_ssi"]].append(ne_solr)
+ end
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "ORG" }.each do |ne_solr|
+ output[:ORG][ne_solr["linked_entity_ssi"]] = [] unless output[:ORG].has_key? ne_solr["linked_entity_ssi"]
+ output[:ORG][ne_solr["linked_entity_ssi"]].append(ne_solr)
+ end
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "HumanProd" }.each do |ne_solr|
+ output[:HumanProd][ne_solr["linked_entity_ssi"]] = [] unless output[:HumanProd].has_key? ne_solr["linked_entity_ssi"]
+ output[:HumanProd][ne_solr["linked_entity_ssi"]].append(ne_solr)
end
+ output
+ end
end
diff --git a/app/models/experiment.rb b/app/models/experiment.rb
index 4d46f6b504bea695e3546397de410da66919dd10..827b2b2458aad84229368d15987fc81364c60ce9 100644
--- a/app/models/experiment.rb
+++ b/app/models/experiment.rb
@@ -1,105 +1,103 @@
class Experiment < ActiveRecord::Base
+ belongs_to :user, optional: false
+ validates :title, length: { minimum: 1 }
- belongs_to :user, optional: false
- validates :title, length: { minimum: 1 }
-
- def add_tool(parent_id, tool)
- if parent_id != 0
- self.locate_tool(self.description, parent_id) do |t|
- t['children'] << tool.to_h
- end
- else
- self.description['children'] << tool.to_h
- end
+ def add_tool(parent_id, tool)
+ if parent_id != 0
+ self.locate_tool(self.description, parent_id) do |t|
+ t["children"] << tool.to_h
+ end
+ else
+ self.description["children"] << tool.to_h
end
+ end
- def delete_tool(tool_id)
- ids = detach_tool(self.description, nil, tool_id)
- end
+ def delete_tool(tool_id)
+ ids = detach_tool(self.description, nil, tool_id)
+ end
- def load_tools
- ids = gather_ids self.description
- Tool.where(id: ids).pluck(:id, :status, :tool_type, :input_type, :output_type, :parent_id).map do |t|
- [t[0], {id: t[0], status: t[1], type: t[2], input_type: t[3], output_type: t[4], parent_id: t[5]}]
- end.to_h
- end
+ def load_tools
+ ids = gather_ids self.description
+ Tool.where(id: ids).pluck(:id, :status, :tool_type, :input_type, :output_type, :parent_id).map do |t|
+ [t[0], { id: t[0], status: t[1], type: t[2], input_type: t[3], output_type: t[4], parent_id: t[5] }]
+ end.to_h
+ end
- def finished?
- tools = self.load_tools
- tools.values.all? { |t| t[:status] == "finished" }
- end
+ def finished?
+ tools = self.load_tools
+ tools.values.all? { |t| t[:status] == "finished" }
+ end
- def running?
- tools = self.load_tools
- tools.values.any? { |t| t[:status] == "running" }
- end
+ def running?
+ tools = self.load_tools
+ tools.values.any? { |t| t[:status] == "running" }
+ end
- def get_tool_ids
- gather_ids self.description
- end
+ def get_tool_ids
+ gather_ids self.description
+ end
- def continue_from(tool_id)
- locate_tool(self.description, tool_id) do |t|
- tools_to_start = t['children'].map { |c| c['tool']['id'] }
- tools_to_start.each do |tool_id|
- tool = Tool.find(tool_id)
- tool.run(true) if tool.runnable?
- end
- end
+ def continue_from(tool_id)
+ locate_tool(self.description, tool_id) do |t|
+ tools_to_start = t["children"].map { |c| c["tool"]["id"] }
+ tools_to_start.each do |tool_id|
+ tool = Tool.find(tool_id)
+ tool.run(true) if tool.runnable?
+ end
end
+ end
- private
+ private
- def locate_tool(tree_part, tool_id, &block)
- if tree_part.has_key?('tool')
- if tree_part['tool']['id'] == tool_id
- yield tree_part
- return true
- else
- tree_part['children'].each do |subtree|
- return true if locate_tool(subtree, tool_id, &block)
- end
- end
- else
- if tree_part['children'].empty?
- yield tree_part
- end
- tree_part['children'].each do |subtree|
- return true if locate_tool(subtree, tool_id, &block)
- end
+ def locate_tool(tree_part, tool_id, &block)
+ if tree_part.has_key?("tool")
+ if tree_part["tool"]["id"] == tool_id
+ yield tree_part
+ return true
+ else
+ tree_part["children"].each do |subtree|
+ return true if locate_tool(subtree, tool_id, &block)
end
- false
+ end
+ else
+ if tree_part["children"].empty?
+ yield tree_part
+ end
+ tree_part["children"].each do |subtree|
+ return true if locate_tool(subtree, tool_id, &block)
+ end
end
+ false
+ end
- def detach_tool(tree, parent_array, tool_id, &block)
- if tree.has_key?('tool')
- if tree['tool']['id'] == tool_id
- ids = gather_ids(tree)
- parent_array.delete(tree) unless parent_array.nil?
- return ids
- else
- tree['children'].each do |subtree|
- res = detach_tool(subtree, tree['children'], tool_id, &block)
- return res unless res.nil?
- end
- end
- else
- tree['children'].each do |subtree|
- res = detach_tool(subtree, tree['children'], tool_id, &block)
- return res unless res.nil?
- end
+ def detach_tool(tree, parent_array, tool_id, &block)
+ if tree.has_key?("tool")
+ if tree["tool"]["id"] == tool_id
+ ids = gather_ids(tree)
+ parent_array.delete(tree) unless parent_array.nil?
+ return ids
+ else
+ tree["children"].each do |subtree|
+ res = detach_tool(subtree, tree["children"], tool_id, &block)
+ return res unless res.nil?
end
- nil
+ end
+ else
+ tree["children"].each do |subtree|
+ res = detach_tool(subtree, tree["children"], tool_id, &block)
+ return res unless res.nil?
+ end
end
+ nil
+ end
- def gather_ids(tree, ids=[])
- tree['children'].each do |subtree|
- ids.concat(gather_ids(subtree))
- end
- if tree.has_key?('tool')
- ids << tree['tool']['id']
- end
- return ids
+ def gather_ids(tree, ids = [])
+ tree["children"].each do |subtree|
+ ids.concat(gather_ids(subtree))
end
-
+ if tree.has_key?("tool")
+ ids << tree["tool"]["id"]
+ end
+ return ids
+ end
end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index e61754174091d46198b9cf3cadc78afeaec24b36..bb289518e3c8facaeb777731955dd92ea959370b 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -1,103 +1,102 @@
class Issue
+ attr_accessor :id, :title, :date_created, :language, :original_uri, :nb_pages, :all_text, :thumbnail_url, :newspaper, :pages, :articles
- attr_accessor :id, :title, :date_created, :language, :original_uri, :nb_pages, :all_text, :thumbnail_url, :newspaper, :pages, :articles
+ def self.from_solr(id, with_pages = false, with_articles = false)
+ solr_doc = SolrSearcher.get_doc_by_id id
+ Issue.from_solr_doc(solr_doc, with_pages, with_articles)
+ end
- def self.from_solr(id, with_pages=false, with_articles=false)
- solr_doc = SolrSearcher.get_doc_by_id id
- Issue.from_solr_doc(solr_doc, with_pages, with_articles)
+ def self.from_solr_doc(solr_doc, with_pages = false, with_articles = false)
+ i = Issue.new
+ i.id = solr_doc["id"]
+ i.language = solr_doc["language_ssi"]
+ i.newspaper = solr_doc["member_of_collection_ids_ssim"][0]
+ i.title = solr_doc["title_ssi"]
+ i.date_created = solr_doc["date_created_ssi"]
+ i.original_uri = solr_doc["original_uri_ss"]
+ i.nb_pages = solr_doc["member_ids_ssim"].size
+ i.thumbnail_url = solr_doc["thumbnail_url_ss"]
+ i.all_text = solr_doc["all_text_t#{i.language}_siv"]
+ if with_pages
+ i.pages = []
+ solr_doc["member_ids_ssim"].each do |pageid|
+ i.pages << Page.from_solr(pageid)
+ end
end
-
- def self.from_solr_doc(solr_doc, with_pages=false, with_articles=false)
- i = Issue.new
- i.id = solr_doc['id']
- i.language = solr_doc['language_ssi']
- i.newspaper = solr_doc['member_of_collection_ids_ssim'][0]
- i.title = solr_doc['title_ssi']
- i.date_created = solr_doc['date_created_ssi']
- i.original_uri = solr_doc['original_uri_ss']
- i.nb_pages = solr_doc['member_ids_ssim'].size
- i.thumbnail_url = solr_doc['thumbnail_url_ss']
- i.all_text = solr_doc["all_text_t#{i.language}_siv"]
- if with_pages
- i.pages = []
- solr_doc['member_ids_ssim'].each do |pageid|
- i.pages << Page.from_solr(pageid)
- end
- end
- if with_articles
- i.articles = []
- articles_docs = SolrSearcher.query({q: "*:*", fq: ["from_issue_ssi:#{i.id}", "has_model_ssim:Article"], fl:"*", rows:10000})['response']['docs']
- articles_docs.each do |articles_doc|
- i.articles << Article.from_solr_doc(articles_doc)
- end
- end
- i
+ if with_articles
+ i.articles = []
+ articles_docs = SolrSearcher.query({ q: "*:*", fq: ["from_issue_ssi:#{i.id}", "has_model_ssim:Article"], fl: "*", rows: 10000 })["response"]["docs"]
+ articles_docs.each do |articles_doc|
+ i.articles << Article.from_solr_doc(articles_doc)
+ end
end
+ i
+ end
- def to_solr
- solr_doc = {}
- solr_doc['id'] = self.id
- solr_doc['has_model_ssim'] = 'Issue'
- solr_doc['title_ssi'] = self.title
- solr_doc['date_created_ssi'] = self.date_created
- solr_doc['date_created_dtsi'] = DateTime.parse(self.date_created).strftime('%Y-%m-%dT%H:%M:%SZ')
- solr_doc['language_ssi'] = self.language
- solr_doc['original_uri_ss'] = self.original_uri
- solr_doc['nb_pages_isi'] = self.nb_pages
- solr_doc['thumbnail_url_ss'] = self.thumbnail_url
- solr_doc['member_ids_ssim'] = self.pages.map(&:id)
- solr_doc['year_isi'] = solr_doc['date_created_ssi'][0..3].to_i
- d = DateTime.parse solr_doc["date_created_dtsi"]
- solr_doc['month_isi'] = d.month
- solr_doc['day_isi'] = d.wday
- solr_doc["member_of_collection_ids_ssim"] = self.newspaper
- solr_doc["all_text_t#{self.language}_siv"] = self.all_text
- solr_doc["all_text_unstemmed_t#{self.language}_siv"] = self.all_text
- solr_doc
- end
+ def to_solr
+ solr_doc = {}
+ solr_doc["id"] = self.id
+ solr_doc["has_model_ssim"] = "Issue"
+ solr_doc["title_ssi"] = self.title
+ solr_doc["date_created_ssi"] = self.date_created
+ solr_doc["date_created_dtsi"] = DateTime.parse(self.date_created).strftime("%Y-%m-%dT%H:%M:%SZ")
+ solr_doc["language_ssi"] = self.language
+ solr_doc["original_uri_ss"] = self.original_uri
+ solr_doc["nb_pages_isi"] = self.nb_pages
+ solr_doc["thumbnail_url_ss"] = self.thumbnail_url
+ solr_doc["member_ids_ssim"] = self.pages.map(&:id)
+ solr_doc["year_isi"] = solr_doc["date_created_ssi"][0..3].to_i
+ d = DateTime.parse solr_doc["date_created_dtsi"]
+ solr_doc["month_isi"] = d.month
+ solr_doc["day_isi"] = d.wday
+ solr_doc["member_of_collection_ids_ssim"] = self.newspaper
+ solr_doc["all_text_t#{self.language}_siv"] = self.all_text
+ solr_doc["all_text_unstemmed_t#{self.language}_siv"] = self.all_text
+ solr_doc
+ end
- def get_thumbnail
- if Rails.configuration.iiif_sources[:local].include? self.newspaper
- "https://iiif.newseye.eu/iiif/#{self.newspaper}/#{self.id}_page_1.ptif/full/200,/0/default.jpg"
- elsif Rails.configuration.iiif_sources[:external].include? self.newspaper
- iiif_pages = self.pages.map{ |p| "#{p.iiif_url}/info.json" } # to change
- elsif Rails.configuration.iiif_sources[:external_onb].include? self.newspaper
- iiif_pages = self.pages.map{ |p| "#{p.iiif_url}/info.json" } # to change
- end
+ def get_thumbnail
+ if Rails.configuration.iiif_sources[:local].include? self.newspaper
+ "https://iiif.newseye.eu/iiif/#{self.newspaper}/#{self.id}_page_1.ptif/full/200,/0/default.jpg"
+ elsif Rails.configuration.iiif_sources[:external].include? self.newspaper
+ iiif_pages = self.pages.map { |p| "#{p.iiif_url}/info.json" } # to change
+ elsif Rails.configuration.iiif_sources[:external_onb].include? self.newspaper
+ iiif_pages = self.pages.map { |p| "#{p.iiif_url}/info.json" } # to change
end
+ end
- def get_iiif_urls
- if Rails.configuration.iiif_sources[:local].include? self.newspaper
- iiif_pages = self.pages.map do |p|
- "https://iiif.newseye.eu/iiif/#{self.newspaper}/#{self.id}_page_#{p.page_number}.ptif/info.json"
- end
- elsif Rails.configuration.iiif_sources[:external].include? self.newspaper
- iiif_pages = self.pages.map{ |p| "#{p.iiif_url}/info.json" }
- elsif Rails.configuration.iiif_sources[:external_onb].include? self.newspaper
- iiif_pages = self.pages.map{ |p| "#{p.iiif_url}/info.json" }
- end
- iiif_pages
+ def get_iiif_urls
+ if Rails.configuration.iiif_sources[:local].include? self.newspaper
+ iiif_pages = self.pages.map do |p|
+ "https://iiif.newseye.eu/iiif/#{self.newspaper}/#{self.id}_page_#{p.page_number}.ptif/info.json"
+ end
+ elsif Rails.configuration.iiif_sources[:external].include? self.newspaper
+ iiif_pages = self.pages.map { |p| "#{p.iiif_url}/info.json" }
+ elsif Rails.configuration.iiif_sources[:external_onb].include? self.newspaper
+ iiif_pages = self.pages.map { |p| "#{p.iiif_url}/info.json" }
end
+ iiif_pages
+ end
- def self.named_entities(issue_id)
- nems = SolrSearcher.query({q:"issue_id_ssi:#{issue_id}", rows: 1000000})['response']['docs']
- output = {LOC: {}, PER: {}, ORG: {}, HumanProd: {}}
- nems.select {|ne_solr| ne_solr['type_ssi'] == "LOC"}.each do |ne_solr|
- output[:LOC][ne_solr['linked_entity_ssi']] = [] unless output[:LOC].has_key? ne_solr['linked_entity_ssi']
- output[:LOC][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- nems.select {|ne_solr| ne_solr['type_ssi'] == "PER"}.each do |ne_solr|
- output[:PER][ne_solr['linked_entity_ssi']] = [] unless output[:PER].has_key? ne_solr['linked_entity_ssi']
- output[:PER][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- nems.select {|ne_solr| ne_solr['type_ssi'] == "ORG"}.each do |ne_solr|
- output[:ORG][ne_solr['linked_entity_ssi']] = [] unless output[:ORG].has_key? ne_solr['linked_entity_ssi']
- output[:ORG][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- nems.select {|ne_solr| ne_solr['type_ssi'] == "HumanProd"}.each do |ne_solr|
- output[:HumanProd][ne_solr['linked_entity_ssi']] = [] unless output[:HumanProd].has_key? ne_solr['linked_entity_ssi']
- output[:HumanProd][ne_solr['linked_entity_ssi']].append(ne_solr)
- end
- output
+ def self.named_entities(issue_id)
+ nems = SolrSearcher.query({ q: "issue_id_ssi:#{issue_id}", rows: 1000000 })["response"]["docs"]
+ output = { LOC: {}, PER: {}, ORG: {}, HumanProd: {} }
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "LOC" }.each do |ne_solr|
+ output[:LOC][ne_solr["linked_entity_ssi"]] = [] unless output[:LOC].has_key? ne_solr["linked_entity_ssi"]
+ output[:LOC][ne_solr["linked_entity_ssi"]].append(ne_solr)
+ end
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "PER" }.each do |ne_solr|
+ output[:PER][ne_solr["linked_entity_ssi"]] = [] unless output[:PER].has_key? ne_solr["linked_entity_ssi"]
+ output[:PER][ne_solr["linked_entity_ssi"]].append(ne_solr)
+ end
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "ORG" }.each do |ne_solr|
+ output[:ORG][ne_solr["linked_entity_ssi"]] = [] unless output[:ORG].has_key? ne_solr["linked_entity_ssi"]
+ output[:ORG][ne_solr["linked_entity_ssi"]].append(ne_solr)
+ end
+ nems.select { |ne_solr| ne_solr["type_ssi"] == "HumanProd" }.each do |ne_solr|
+ output[:HumanProd][ne_solr["linked_entity_ssi"]] = [] unless output[:HumanProd].has_key? ne_solr["linked_entity_ssi"]
+ output[:HumanProd][ne_solr["linked_entity_ssi"]].append(ne_solr)
end
-end
\ No newline at end of file
+ output
+ end
+end
diff --git a/app/models/notification.rb b/app/models/notification.rb
index d2bb227c3ca5349e3e0481b3b85e5cb34c289e34..be5427653e6b7f8da4a4c42692410e518cf06258 100644
--- a/app/models/notification.rb
+++ b/app/models/notification.rb
@@ -1,6 +1,3 @@
class Notification < ActiveRecord::Base
-
- belongs_to :user, optional: false
-
-
-end
\ No newline at end of file
+ belongs_to :user, optional: false
+end
diff --git a/app/models/page.rb b/app/models/page.rb
index 87f311c37a796a06db41c14857e1344943f0654d..0ba49faa195ed7b4de69a1d68cc5fdcd1d48c236 100644
--- a/app/models/page.rb
+++ b/app/models/page.rb
@@ -1,31 +1,30 @@
class Page
+ attr_accessor :id, :page_number, :width, :height, :mime_type, :iiif_url, :ocr_path, :image_path
- attr_accessor :id, :page_number, :width, :height, :mime_type, :iiif_url, :ocr_path, :image_path
+ def self.from_solr(id)
+ attrs = SolrSearcher.get_doc_by_id id
+ p = Page.new
+ p.id = attrs["id"]
+ p.page_number = attrs["page_number_isi"]
+ p.width = attrs["width_isi"]
+ p.height = attrs["height_isi"]
+ p.mime_type = attrs["mime_type_ssi"]
+ p.iiif_url = attrs["iiif_url_ss"]
+ p.ocr_path = attrs["ocr_path_ss"]
+ p.image_path = attrs["image_path_ss"] if attrs["image_path_ss"]
+ p
+ end
- def self.from_solr id
- attrs = SolrSearcher.get_doc_by_id id
- p = Page.new
- p.id = attrs['id']
- p.page_number = attrs['page_number_isi']
- p.width = attrs['width_isi']
- p.height = attrs['height_isi']
- p.mime_type = attrs['mime_type_ssi']
- p.iiif_url = attrs['iiif_url_ss']
- p.ocr_path = attrs['ocr_path_ss']
- p.image_path = attrs['image_path_ss'] if attrs['image_path_ss']
- p
- end
-
- def to_solr
- solr_doc = {}
- solr_doc['id'] = self.id
- solr_doc['has_model_ssim'] = 'PageFileSet'
- solr_doc['page_number_isi'] = self.page_number
- solr_doc['width_isi'] = self.width
- solr_doc['height_isi'] = self.height
- solr_doc['mime_type_ssi'] = self.mime_type
- solr_doc['iiif_url_ss'] = self.iiif_url
- solr_doc['ocr_path_ss'] = self.ocr_path
- solr_doc
- end
-end
\ No newline at end of file
+ def to_solr
+ solr_doc = {}
+ solr_doc["id"] = self.id
+ solr_doc["has_model_ssim"] = "PageFileSet"
+ solr_doc["page_number_isi"] = self.page_number
+ solr_doc["width_isi"] = self.width
+ solr_doc["height_isi"] = self.height
+ solr_doc["mime_type_ssi"] = self.mime_type
+ solr_doc["iiif_url_ss"] = self.iiif_url
+ solr_doc["ocr_path_ss"] = self.ocr_path
+ solr_doc
+ end
+end
diff --git a/app/models/solr_query.rb b/app/models/solr_query.rb
index 3e5edd7a6427a19154c15c217b6af02285dffaa0..80ae868b4b9799216d6b8d5beec0169c290950a2 100644
--- a/app/models/solr_query.rb
+++ b/app/models/solr_query.rb
@@ -1,54 +1,52 @@
class SolrQuery
+ attr_accessor :defType, :sort, :start, :rows, :fq, :fl, # common parameters
+ :q, :q_dot_alt, :qf, :mm, :pf, :ps, :qs, :tie, :bq, :bf, # Dismax parameters
+ :sow, :mm_dot_autorelax, :boost, :lowercaseOperators, :pf2, :ps2, :pf3, :ps3, :stopwords, :uf, # Edismax parameters
+ :facet, :facet_dot_field, :facet_dot_threads,
+ :hl,
+ :mlt
- attr_accessor :defType, :sort, :start, :rows, :fq, :fl, # common parameters
- :q, :q_dot_alt, :qf, :mm, :pf, :ps, :qs, :tie, :bq, :bf, # Dismax parameters
- :sow, :mm_dot_autorelax, :boost, :lowercaseOperators, :pf2, :ps2, :pf3, :ps3, :stopwords, :uf, # Edismax parameters
- :facet, :facet_dot_field, :facet_dot_threads,
- :hl,
- :mlt
+ def initialize(search_type)
+ @defType = "edismax"
+ @sort = "score desc"
+ @start = 0
+ @rows = 10
+ # @fq = ["has_model_ssim:(Article OR Issue)"]
+ @fq = ["has_model_ssim:(Article)"]
+ @fl = "*,score"
+ @q = "*:*"
+ @q_dot_alt = "*:*"
+ @qf = I18n.t("newspapers.solr_fields").select { |k, v| k.start_with?(search_type == "stemmed" ? "text_stemmed" : "text_exact") }.values # or text_stemmed
+ @mm = 1
+ @pf = ""
+ @ps = ""
+ @qs = ""
+ @tie = 0.1
+ @bq = ""
+ @bf = ""
+ @hl = true
+ @hl_dot_fl = @qf
- def initialize search_type
- @defType = 'edismax'
- @sort = 'score desc'
- @start = 0
- @rows = 10
- # @fq = ["has_model_ssim:(Article OR Issue)"]
- @fq = ["has_model_ssim:(Article)"]
- @fl = '*,score'
- @q = '*:*'
- @q_dot_alt = '*:*'
- @qf = I18n.t("newspapers.solr_fields").select{|k,v| k.start_with?( search_type=="stemmed" ? "text_stemmed" : "text_exact") }.values # or text_stemmed
- @mm = 1
- @pf = ""
- @ps = ""
- @qs = ""
- @tie = 0.1
- @bq = ""
- @bf = ""
- @hl = true
- @hl_dot_fl = @qf
-
- @json_dot_facet = {}
- I18n.t("newspapers.solr_fields").values_at(:language, :newspaper).each do |f|
- @json_dot_facet[f] = { terms: { field: f, limit: 15, numBuckets: true} }
- end
- I18n.t("newspapers.solr_fields").values_at(:date).each do |f|
- @json_dot_facet[f] = { terms: { field: f, limit: -1, numBuckets: true} }
- end
- I18n.t("newspapers.solr_fields").values_at(:month, :day).each do |f|
- @json_dot_facet[f] = { terms: { field: f, limit: 15, numBuckets: true, sort: {index: "asc"}} }
- end
- I18n.t("newspapers.solr_fields").values_at(:persons, :locations, :organisations, :human_productions).each do |f|
- @json_dot_facet[f] = { terms: { field: f, limit: 15, numBuckets: true} }
- end
- @json_dot_facet["min_date"] = "min(date_created_dtsi)"
- @json_dot_facet["max_date"] = "max(date_created_dtsi)"
+ @json_dot_facet = {}
+ I18n.t("newspapers.solr_fields").values_at(:language, :newspaper).each do |f|
+ @json_dot_facet[f] = { terms: { field: f, limit: 15, numBuckets: true } }
end
-
- def to_params
- p = self.instance_values.select {|k,v| v != "" and !v.nil?}.transform_keys{|k| k.gsub('_dot_','.')}.with_indifferent_access
- p["json.facet"] = p["json.facet"].to_json
- p
+ I18n.t("newspapers.solr_fields").values_at(:date).each do |f|
+ @json_dot_facet[f] = { terms: { field: f, limit: -1, numBuckets: true } }
+ end
+ I18n.t("newspapers.solr_fields").values_at(:month, :day).each do |f|
+ @json_dot_facet[f] = { terms: { field: f, limit: 15, numBuckets: true, sort: { index: "asc" } } }
+ end
+ I18n.t("newspapers.solr_fields").values_at(:persons, :locations, :organisations, :human_productions).each do |f|
+ @json_dot_facet[f] = { terms: { field: f, limit: 15, numBuckets: true } }
end
+ @json_dot_facet["min_date"] = "min(date_created_dtsi)"
+ @json_dot_facet["max_date"] = "max(date_created_dtsi)"
+ end
-end
\ No newline at end of file
+ def to_params
+ p = self.instance_values.select { |k, v| v != "" and !v.nil? }.transform_keys { |k| k.gsub("_dot_", ".") }.with_indifferent_access
+ p["json.facet"] = p["json.facet"].to_json
+ p
+ end
+end
diff --git a/app/models/solr_searcher.rb b/app/models/solr_searcher.rb
index e2c4add2ac0b718e727d7406fe09270e098aa5b2..0ba18632ca8e6d595811c96e28600105feca2ec1 100644
--- a/app/models/solr_searcher.rb
+++ b/app/models/solr_searcher.rb
@@ -1,26 +1,25 @@
class SolrSearcher
- include AbstractSearcher
+ include AbstractSearcher
- @@connection = false
+ @@connection = false
- def self.query params
- connect unless @@connection
- puts "[SolrSearcher.Query] #{params.to_json}\n" if Rails.env == "development"
- @@connection.send_and_receive("select", data: params, method: :post)
- end
+ def self.query(params)
+ connect unless @@connection
+ puts "[SolrSearcher.Query] #{params.to_json}\n" if Rails.env == "development"
+ @@connection.send_and_receive("select", data: params, method: :post)
+ end
- def self.connect
- @@connection = RSolr.connect(url: Rails.configuration.solr['url']) unless @@connection
- end
+ def self.connect
+ @@connection = RSolr.connect(url: Rails.configuration.solr["url"]) unless @@connection
+ end
- def self.get_doc_by_id(id)
- connect unless @@connection
- docs = @@connection.send_and_receive("select", data: {q: "id:#{id}"}, method: :post)['response']['docs']
- if docs.empty?
- nil
- else
- docs[0]
- end
+ def self.get_doc_by_id(id)
+ connect unless @@connection
+ docs = @@connection.send_and_receive("select", data: { q: "id:#{id}" }, method: :post)["response"]["docs"]
+ if docs.empty?
+ nil
+ else
+ docs[0]
end
-
-end
\ No newline at end of file
+ end
+end
diff --git a/app/models/tool.rb b/app/models/tool.rb
index 35b0b6b528fb56e36394d0db1fe797bc5bbbc222..599c79670dfa0c5fba18a0e8e9a5043d1a13b98f 100644
--- a/app/models/tool.rb
+++ b/app/models/tool.rb
@@ -1,22 +1,20 @@
class Tool < ActiveRecord::Base
+ belongs_to :experiment, optional: false
- belongs_to :experiment, optional: false
-
- def to_h
- {
+ def to_h
+ {
"tool": {
- "id": self.id
+ "id": self.id,
},
- "children": []
+ "children": [],
}
- end
-
- def runnable?
- self.status == "configured" && (self.parent_id.nil? || Tool.find(self.parent_id).status == "finished")
- end
+ end
- def run(continue=false)
- "#{self.tool_type}_worker".camelize.constantize.perform_async(self.id, self.experiment.user.id, self.experiment.id, self.tool_type, self.parameters, continue)
- end
+ def runnable?
+ self.status == "configured" && (self.parent_id.nil? || Tool.find(self.parent_id).status == "finished")
+ end
+ def run(continue = false)
+ "#{self.tool_type}_worker".camelize.constantize.perform_async(self.id, self.experiment.user.id, self.experiment.id, self.tool_type, self.parameters, continue)
+ end
end
diff --git a/app/models/user.rb b/app/models/user.rb
index ae04d79f08b8ca935404e2f108d56c041aa78dca..f02f51be431faac33063b78e75dded039f33f586 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -1,28 +1,28 @@
class User < ApplicationRecord
- attribute :labs_user_id, presence: true, unique: true
+ attribute :labs_user_id, presence: true, unique: true
- has_many :experiments
- has_many :datasets
- has_many :notifications
- has_many :compound_articles
- has_many :active_sessions
+ has_many :experiments
+ has_many :datasets
+ has_many :notifications
+ has_many :compound_articles
+ has_many :active_sessions
- def datasets_with_doc doc_id
- self.datasets.map do |dataset|
- [dataset.id, dataset.title] if dataset.contains doc_id.to_s
- end.delete_if(&:nil?)
- end
+ def datasets_with_doc(doc_id)
+ self.datasets.map do |dataset|
+ [dataset.id, dataset.title] if dataset.contains doc_id.to_s
+ end.delete_if(&:nil?)
+ end
- def compounds_by_issue
- out = {}
- self.compound_articles.each do |compound_article|
- out[compound_article.issue_id] = [] unless out.has_key? compound_article.issue_id
- out[compound_article.issue_id] << compound_article
- end
- out
+ def compounds_by_issue
+ out = {}
+ self.compound_articles.each do |compound_article|
+ out[compound_article.issue_id] = [] unless out.has_key? compound_article.issue_id
+ out[compound_article.issue_id] << compound_article
end
+ out
+ end
- def researcher?
- Rails.configuration.auths['emails'].include? self.email
- end
+ def researcher?
+ Rails.configuration.auths["emails"].include? self.email
+ end
end
diff --git a/app/workers/export_dataset_worker.rb b/app/workers/export_dataset_worker.rb
index 8dad51bcb84c6e39cf684c774ee10fbe7d6fffd4..a0a760b5858893f7fdb5c11bc72df345ecf52eda 100644
--- a/app/workers/export_dataset_worker.rb
+++ b/app/workers/export_dataset_worker.rb
@@ -1,88 +1,88 @@
require "zip"
class ExportDatasetWorker
- include Sidekiq::Worker
- include ActionView::Helpers::FormOptionsHelper
+ include Sidekiq::Worker
+ include ActionView::Helpers::FormOptionsHelper
- def perform(user_id, dataset_id, export_type)
- dataset = Dataset.find(dataset_id)
- file = Tempfile.new(["export_#{dataset.title.parameterize(separator: '_')}_", ".#{export_type}"], "tmp")
- to_write = []
- named_entities = dataset.named_entities
- named_entities = named_entities.values.map{|h| h.values }.flatten
- documents = dataset.fetch_paginated_documents(1, 100, "default", "asc", "all", recursive=true)
- documents[:docs].map do |doc|
- case export_type
- when "json"
- lang = doc.language
- thumb = doc.thumbnail_url
- if doc.is_a?(CompoundArticle)
- doc_type = "compound"
- entities = named_entities.select{ |ne| doc.parts.include? ne['article_id_ssi'] }
- elsif doc.is_a?(Article)
- doc_type = "article"
- entities = named_entities.select{ |ne| doc.id == ne['article_id_ssi'] }
- elsif doc.is_a?(Issue)
- doc_type = "issue"
- entities = named_entities.select{ |ne| doc.id == ne['issue_id_ssi'] }
- end
- entities = entities.map do |ne|
- {
- mention: ne['mention_ssi'],
- indexStart: ne['article_index_start_isi'],
- indexEnd: ne['article_index_end_isi'],
- stance: if ne['stance_fsi'] == 0
- "neutral"
- else
- ne['stance_fsi'] > 0 ? "positive" : "negative"
- end,
- linked_entity_url: ne['linked_entity_ssi'] == "" || ne['linked_entity_ssi'].nil? ? nil : "https://www.wikidata.org/wiki/#{ne['linked_entity_ssi'].split('_')[-1]}"
+ def perform(user_id, dataset_id, export_type)
+ dataset = Dataset.find(dataset_id)
+ file = Tempfile.new(["export_#{dataset.title.parameterize(separator: "_")}_", ".#{export_type}"], "tmp")
+ to_write = []
+ named_entities = dataset.named_entities
+ named_entities = named_entities.values.map { |h| h.values }.flatten
+ documents = dataset.fetch_paginated_documents(1, 100, "default", "asc", "all", recursive = true)
+ documents[:docs].map do |doc|
+ case export_type
+ when "json"
+ lang = doc.language
+ thumb = doc.thumbnail_url
+ if doc.is_a?(CompoundArticle)
+ doc_type = "compound"
+ entities = named_entities.select { |ne| doc.parts.include? ne["article_id_ssi"] }
+ elsif doc.is_a?(Article)
+ doc_type = "article"
+ entities = named_entities.select { |ne| doc.id == ne["article_id_ssi"] }
+ elsif doc.is_a?(Issue)
+ doc_type = "issue"
+ entities = named_entities.select { |ne| doc.id == ne["issue_id_ssi"] }
+ end
+ entities = entities.map do |ne|
+ {
+ mention: ne["mention_ssi"],
+ indexStart: ne["article_index_start_isi"],
+ indexEnd: ne["article_index_end_isi"],
+ stance: if ne["stance_fsi"] == 0
+ "neutral"
+ else
+ ne["stance_fsi"] > 0 ? "positive" : "negative"
+ end,
+ linked_entity_url: ne["linked_entity_ssi"] == "" || ne["linked_entity_ssi"].nil? ? nil : "https://www.wikidata.org/wiki/#{ne["linked_entity_ssi"].split("_")[-1]}",
}
- end
- unless under_copyright(lang, doc.date_created, User.find(user_id))
- to_write << { id: doc.id,
- type: doc_type,
- language: lang,
- date: doc.date_created,
- newspaper_id: doc.newspaper,
- iiif_url: thumb,
- text: doc.all_text,
- named_entities: entities }
- end
- when "zip"
- filename = "#{doc.date_created}_#{doc.is_a?(CompoundArticle) ? "compound_#{doc.title}" : doc.id}.txt"
- file_content = doc.all_text
- File.open("/tmp/#{filename}", 'w') do |f|
- f.write file_content
- end
- to_write << filename
- end
end
- case export_type
- when "json"
- to_write = {"documents": to_write}
- file.write to_write.to_json
- file.close
- when "zip"
- Zip::File.open(file.path, Zip::File::CREATE) do |zipfile|
- to_write.each do |filename|
- zipfile.add filename, "/tmp/#{filename}" if filename
- end
- end
- to_write.each do |filename|
- File.delete("/tmp/#{filename}") if filename
- end
+ unless under_copyright(lang, doc.date_created, User.find(user_id))
+ to_write << { id: doc.id,
+ type: doc_type,
+ language: lang,
+ date: doc.date_created,
+ newspaper_id: doc.newspaper,
+ iiif_url: thumb,
+ text: doc.all_text,
+ named_entities: entities }
+ end
+ when "zip"
+ filename = "#{doc.date_created}_#{doc.is_a?(CompoundArticle) ? "compound_#{doc.title}" : doc.id}.txt"
+ file_content = doc.all_text
+ File.open("/tmp/#{filename}", "w") do |f|
+ f.write file_content
end
- content = "Your dataset is ready. Click here to download it.
"
- ActionCable.server.broadcast("notifications.#{user_id}", {
- type: "notify",
- html: ApplicationController.render(partial: "shared/notification", locals: {notif_title: dataset.title, notif_content: content})
- })
+ to_write << filename
+ end
end
-
- def under_copyright(lang, date, user)
- nlf_doc = ["fi", "se"].include? lang
- nlf_under_copyright = Date.parse("1910-12-31T00:00:00Z") <= Date.parse(date)
- nlf_doc and nlf_under_copyright and !user.groups.include? "researcher"
+ case export_type
+ when "json"
+ to_write = { "documents": to_write }
+ file.write to_write.to_json
+ file.close
+ when "zip"
+ Zip::File.open(file.path, Zip::File::CREATE) do |zipfile|
+ to_write.each do |filename|
+ zipfile.add filename, "/tmp/#{filename}" if filename
+ end
+ end
+ to_write.each do |filename|
+ File.delete("/tmp/#{filename}") if filename
+ end
end
-end
\ No newline at end of file
+ content = "Your dataset is ready. Click here to download it.
"
+ ActionCable.server.broadcast("notifications.#{user_id}", {
+ type: "notify",
+ html: ApplicationController.render(partial: "shared/notification", locals: { notif_title: dataset.title, notif_content: content }),
+ })
+ end
+
+ def under_copyright(lang, date, user)
+ nlf_doc = ["fi", "se"].include? lang
+ nlf_under_copyright = Date.parse("1910-12-31T00:00:00Z") <= Date.parse(date)
+ nlf_doc and nlf_under_copyright and !user.groups.include? "researcher"
+ end
+end
diff --git a/app/workers/ngrams_worker.rb b/app/workers/ngrams_worker.rb
index bc3dbee5e62d2c340b8312b6ef406d7f296d1884..348268e27553cd960d76f339e31dc08a866845a8 100644
--- a/app/workers/ngrams_worker.rb
+++ b/app/workers/ngrams_worker.rb
@@ -1,58 +1,59 @@
class NgramsWorker
- include Sidekiq::Worker
+ include Sidekiq::Worker
- def perform(tool_id, user_id, experiment_id, tool_type, tool_parameters, continue=false)
- tool = Tool.find(tool_id)
- tool.status = "running"
- tool.save!
- ActionCable.server.broadcast("notifications.#{user_id}", {
- type: "refresh_display",
- html: ApplicationController.render(partial: "experiment/tree", locals: {experiment: Experiment.find(tool.experiment.id)}),
- message: 'Starting job...' })
- parent_output = Tool.find(tool.parent_id).results
- docs = parent_output["docs"].map{ |doc| doc['text'] }
- n = tool_parameters.select{|t| t['name'] == 'n'}[0]['value'].to_i
- min_freq = tool_parameters.select{|t| t['name'] == 'minimum_frequency'}[0]['value'].to_i
- ngrams = find_ngrams(tool_id, experiment_id, user_id, docs, n, min_freq)
- tool.results = {type:"ngrams", ngrams: ngrams}
- tool.status = "finished"
- tool.save!
- experiment = Experiment.find(tool.experiment.id)
- out = {
- type: "refresh_display",
- html: ApplicationController.render(partial: "experiment/tree", locals: {experiment: experiment}),
- message: 'Done.'
- }
- ActionCable.server.broadcast("notifications.#{user_id}", out)
- if continue
- experiment.continue_from(tool_id)
- end
- if experiment.finished?
- out = {
- type: "experiment_finished",
- message: 'Experiment has finished running.'
- }
- ActionCable.server.broadcast("notifications.#{user_id}", out)
- end
+ def perform(tool_id, user_id, experiment_id, tool_type, tool_parameters, continue = false)
+ tool = Tool.find(tool_id)
+ tool.status = "running"
+ tool.save!
+ ActionCable.server.broadcast("notifications.#{user_id}", {
+ type: "refresh_display",
+ html: ApplicationController.render(partial: "experiment/tree", locals: { experiment: Experiment.find(tool.experiment.id) }),
+ message: "Starting job...",
+ })
+ parent_output = Tool.find(tool.parent_id).results
+ docs = parent_output["docs"].map { |doc| doc["text"] }
+ n = tool_parameters.select { |t| t["name"] == "n" }[0]["value"].to_i
+ min_freq = tool_parameters.select { |t| t["name"] == "minimum_frequency" }[0]["value"].to_i
+ ngrams = find_ngrams(tool_id, experiment_id, user_id, docs, n, min_freq)
+ tool.results = { type: "ngrams", ngrams: ngrams }
+ tool.status = "finished"
+ tool.save!
+ experiment = Experiment.find(tool.experiment.id)
+ out = {
+ type: "refresh_display",
+ html: ApplicationController.render(partial: "experiment/tree", locals: { experiment: experiment }),
+ message: "Done.",
+ }
+ ActionCable.server.broadcast("notifications.#{user_id}", out)
+ if continue
+ experiment.continue_from(tool_id)
end
+ if experiment.finished?
+ out = {
+ type: "experiment_finished",
+ message: "Experiment has finished running.",
+ }
+ ActionCable.server.broadcast("notifications.#{user_id}", out)
+ end
+ end
- def find_ngrams(tool_id, experiment_id, user_id, documents, n, minimum_frequency)
- total = {}
- documents.each_with_index do |document, idx|
- out = {
- type: "completion_rate",
- tool_id: tool_id,
- experiment_id: experiment_id,
- completion: ((idx/(documents.size).to_f)*100).to_i
- }
- ActionCable.server.broadcast("notifications.#{user_id}", out) if idx%20 == 0
- ngrams = document.split.each_cons(n).to_a
- ngrams.reject! { |w1, w2| w1 !~ /^\w+/ || w2 !~ /^\w+/ }
- ngrams.map!{ |ngram| ngram.join(' ') }
- total.merge!( ngrams.each_with_object(Hash.new(0)) do |word, obj|
- obj[word.downcase] += 1
- end)
- end
- total.sort_by { |k, v| -v }.reject { |k, v| v < minimum_frequency }
+ def find_ngrams(tool_id, experiment_id, user_id, documents, n, minimum_frequency)
+ total = {}
+ documents.each_with_index do |document, idx|
+ out = {
+ type: "completion_rate",
+ tool_id: tool_id,
+ experiment_id: experiment_id,
+ completion: ((idx / (documents.size).to_f) * 100).to_i,
+ }
+ ActionCable.server.broadcast("notifications.#{user_id}", out) if idx % 20 == 0
+ ngrams = document.split.each_cons(n).to_a
+ ngrams.reject! { |w1, w2| w1 !~ /^\w+/ || w2 !~ /^\w+/ }
+ ngrams.map! { |ngram| ngram.join(" ") }
+ total.merge!(ngrams.each_with_object(Hash.new(0)) do |word, obj|
+ obj[word.downcase] += 1
+ end)
end
+ total.sort_by { |k, v| -v }.reject { |k, v| v < minimum_frequency }
+ end
end
diff --git a/app/workers/preprocess_worker.rb b/app/workers/preprocess_worker.rb
index f63433446efbb110e91fd48326d281096e6a1fc9..ee740c17a00ec0be6f1b75c341f0e3b9f7732082 100644
--- a/app/workers/preprocess_worker.rb
+++ b/app/workers/preprocess_worker.rb
@@ -1,55 +1,56 @@
class PreprocessWorker
- include Sidekiq::Worker
+ include Sidekiq::Worker
- def perform(tool_id, user_id, experiment_id, tool_type, tool_parameters, continue=false)
- tool = Tool.find(tool_id)
- tool.status = "running"
- tool.save!
- ActionCable.server.broadcast("notifications.#{user_id}", {
- type: "refresh_display",
- html: ApplicationController.render(partial: "experiment/tree", locals: {experiment: Experiment.find(tool.experiment.id)}),
- message: 'Starting job...' })
- parent_output = Tool.find(tool.parent_id).results
- docs = parent_output["docs"]
- docs = docs.each_with_index.map do |doc, idx|
- out = {
- type: "completion_rate",
- tool_id: tool.id,
- experiment_id: experiment_id,
- completion: ((idx/(docs.size).to_f)*100).to_i
- }
- ActionCable.server.broadcast("notifications.#{user_id}", out) if idx%20 == 0
+ def perform(tool_id, user_id, experiment_id, tool_type, tool_parameters, continue = false)
+ tool = Tool.find(tool_id)
+ tool.status = "running"
+ tool.save!
+ ActionCable.server.broadcast("notifications.#{user_id}", {
+ type: "refresh_display",
+ html: ApplicationController.render(partial: "experiment/tree", locals: { experiment: Experiment.find(tool.experiment.id) }),
+ message: "Starting job...",
+ })
+ parent_output = Tool.find(tool.parent_id).results
+ docs = parent_output["docs"]
+ docs = docs.each_with_index.map do |doc, idx|
+ out = {
+ type: "completion_rate",
+ tool_id: tool.id,
+ experiment_id: experiment_id,
+ completion: ((idx / (docs.size).to_f) * 100).to_i,
+ }
+ ActionCable.server.broadcast("notifications.#{user_id}", out) if idx % 20 == 0
- doc['text'] = PragmaticTokenizer::Tokenizer.new(
- language: doc['language'],
- remove_stop_words: tool_parameters.select{|t| t['name'] == 'stopwords'}[0]['value'],
- punctuation: tool_parameters.select{|t| t['name'] == 'punctuation'}[0]['value'] ? "none" : "all",
- numbers: tool_parameters.select{|t| t['name'] == 'lowercase'}[0]['value'] ? "none" : "all",
- clean: true,
- downcase: tool_parameters.select{|t| t['name'] == 'lowercase'}[0]['value'],
- minimum_length: 3
- ).tokenize(doc['text']).join(' ')
- doc
- end
- tool.results = {type:"documents", docs: docs}
- tool.status = "finished"
- tool.save!
- experiment = Experiment.find(tool.experiment.id)
- out = {
- type: "refresh_display",
- html: ApplicationController.render(partial: "experiment/tree", locals: {experiment: experiment}),
- message: 'Done.'
- }
- ActionCable.server.broadcast("notifications.#{user_id}", out)
- if continue
- experiment.continue_from(tool_id)
- end
- if experiment.finished?
- out = {
- type: "experiment_finished",
- message: 'Experiment has finished running.'
- }
- ActionCable.server.broadcast("notifications.#{user_id}", out)
- end
+ doc["text"] = PragmaticTokenizer::Tokenizer.new(
+ language: doc["language"],
+ remove_stop_words: tool_parameters.select { |t| t["name"] == "stopwords" }[0]["value"],
+ punctuation: tool_parameters.select { |t| t["name"] == "punctuation" }[0]["value"] ? "none" : "all",
+ numbers: tool_parameters.select { |t| t["name"] == "lowercase" }[0]["value"] ? "none" : "all",
+ clean: true,
+ downcase: tool_parameters.select { |t| t["name"] == "lowercase" }[0]["value"],
+ minimum_length: 3,
+ ).tokenize(doc["text"]).join(" ")
+ doc
end
+ tool.results = { type: "documents", docs: docs }
+ tool.status = "finished"
+ tool.save!
+ experiment = Experiment.find(tool.experiment.id)
+ out = {
+ type: "refresh_display",
+ html: ApplicationController.render(partial: "experiment/tree", locals: { experiment: experiment }),
+ message: "Done.",
+ }
+ ActionCable.server.broadcast("notifications.#{user_id}", out)
+ if continue
+ experiment.continue_from(tool_id)
+ end
+ if experiment.finished?
+ out = {
+ type: "experiment_finished",
+ message: "Experiment has finished running.",
+ }
+ ActionCable.server.broadcast("notifications.#{user_id}", out)
+ end
+ end
end
diff --git a/app/workers/search_to_dataset_worker.rb b/app/workers/search_to_dataset_worker.rb
index ce1da67b2b33c9c5ec362162eea31d6687e622e8..231529423a761d68d8681ab4de63fc1c32fdf028 100644
--- a/app/workers/search_to_dataset_worker.rb
+++ b/app/workers/search_to_dataset_worker.rb
@@ -1,34 +1,34 @@
class SearchToDatasetWorker
- include Sidekiq::Worker
- include ActionView::Helpers::FormOptionsHelper
+ include Sidekiq::Worker
+ include ActionView::Helpers::FormOptionsHelper
- def perform(user_id, dataset_id, search_params)
- puts "### #{search_params}"
- dataset = Dataset.find(dataset_id)
- search_params['fl'] = 'id'
- search_params['facet'] = false
- search_params['rows'] = 100
- search_params['start'] = 0
- doc_ids = []
- res = SolrSearcher.query search_params
- numFound = res['response']['numFound']
- doc_ids.concat res['response']['docs'].map{|d| d['id']}
- while(doc_ids.size < numFound)
- search_params['start'] += 100
- res = SolrSearcher.query search_params
- numFound = res['response']['numFound']
- doc_ids.concat res['response']['docs'].map{|d| d['id']}
- end
- existing = dataset.add_documents doc_ids
- nb_docs_added = doc_ids.size - existing.size
- content = "#{nb_docs_added} document#{nb_docs_added > 1 ? "s were" : " was"} added to your dataset \"#{dataset.title}\"
"
- content.concat "#{existing.size} document#{existing.size > 1 ? "s" : ""} already exist in this dataset.
" unless existing.empty?
- # TODO: next line may cause bugs with the working dataset
- dataset_options = options_for_select(User.find(user_id).datasets.map{|d| ["#{d.title} (#{d.documents.size} docs)", d.id]})
- ActionCable.server.broadcast("notifications.#{user_id}", {
- type: "notify",
- html: ApplicationController.render(partial: "shared/notification", locals: {notif_title: dataset.title, notif_content: content}),
- dataset_options: dataset_options })
+ def perform(user_id, dataset_id, search_params)
+ puts "### #{search_params}"
+ dataset = Dataset.find(dataset_id)
+ search_params["fl"] = "id"
+ search_params["facet"] = false
+ search_params["rows"] = 100
+ search_params["start"] = 0
+ doc_ids = []
+ res = SolrSearcher.query search_params
+ numFound = res["response"]["numFound"]
+ doc_ids.concat res["response"]["docs"].map { |d| d["id"] }
+ while (doc_ids.size < numFound)
+ search_params["start"] += 100
+ res = SolrSearcher.query search_params
+ numFound = res["response"]["numFound"]
+ doc_ids.concat res["response"]["docs"].map { |d| d["id"] }
end
-
-end
\ No newline at end of file
+ existing = dataset.add_documents doc_ids
+ nb_docs_added = doc_ids.size - existing.size
+ content = "#{nb_docs_added} document#{nb_docs_added > 1 ? "s were" : " was"} added to your dataset \"#{dataset.title}\"
"
+ content.concat "#{existing.size} document#{existing.size > 1 ? "s" : ""} already exist in this dataset.
" unless existing.empty?
+ # TODO: next line may cause bugs with the working dataset
+ dataset_options = options_for_select(User.find(user_id).datasets.map { |d| ["#{d.title} (#{d.documents.size} docs)", d.id] })
+ ActionCable.server.broadcast("notifications.#{user_id}", {
+ type: "notify",
+ html: ApplicationController.render(partial: "shared/notification", locals: { notif_title: dataset.title, notif_content: content }),
+ dataset_options: dataset_options,
+ })
+ end
+end
diff --git a/app/workers/source_dataset_worker.rb b/app/workers/source_dataset_worker.rb
index bda0ec497511d3a0e16544d3971cbd98e5f996f2..d303b8f9477a826953c3abcbd381fc29a7d7ed35 100644
--- a/app/workers/source_dataset_worker.rb
+++ b/app/workers/source_dataset_worker.rb
@@ -1,62 +1,63 @@
class SourceDatasetWorker
- include Sidekiq::Worker
+ include Sidekiq::Worker
- def perform(tool_id, user_id, experiment_id, tool_type, tool_parameters, continue=false)
- tool = Tool.find(tool_id)
- tool.status = "running"
- tool.save!
- ActionCable.server.broadcast("notifications.#{user_id}", {
- type: "refresh_display",
- html: ApplicationController.render(partial: "experiment/tree", locals: {experiment: Experiment.find(tool.experiment.id)}),
- message: 'Starting job...' })
- docs = fetch_docs_from_dataset(tool_id, experiment_id, user_id, tool_parameters.select{|t| t['name'] == 'dataset'}[0]['value'])
- tool.results = {type:"documents", docs: docs}
- tool.status = "finished"
- tool.save!
- experiment = Experiment.find(tool.experiment.id)
- out = {
- type: "refresh_display",
- html: ApplicationController.render(partial: "experiment/tree", locals: {experiment: experiment}),
- message: 'Done.'
- }
- ActionCable.server.broadcast("notifications.#{user_id}", out)
- if continue
- experiment.continue_from(tool_id)
- end
- if experiment.finished?
- out = {
- type: "experiment_finished",
- message: 'Experiment has finished running.'
- }
- ActionCable.server.broadcast("notifications.#{user_id}", out)
- end
+ def perform(tool_id, user_id, experiment_id, tool_type, tool_parameters, continue = false)
+ tool = Tool.find(tool_id)
+ tool.status = "running"
+ tool.save!
+ ActionCable.server.broadcast("notifications.#{user_id}", {
+ type: "refresh_display",
+ html: ApplicationController.render(partial: "experiment/tree", locals: { experiment: Experiment.find(tool.experiment.id) }),
+ message: "Starting job...",
+ })
+ docs = fetch_docs_from_dataset(tool_id, experiment_id, user_id, tool_parameters.select { |t| t["name"] == "dataset" }[0]["value"])
+ tool.results = { type: "documents", docs: docs }
+ tool.status = "finished"
+ tool.save!
+ experiment = Experiment.find(tool.experiment.id)
+ out = {
+ type: "refresh_display",
+ html: ApplicationController.render(partial: "experiment/tree", locals: { experiment: experiment }),
+ message: "Done.",
+ }
+ ActionCable.server.broadcast("notifications.#{user_id}", out)
+ if continue
+ experiment.continue_from(tool_id)
+ end
+ if experiment.finished?
+ out = {
+ type: "experiment_finished",
+ message: "Experiment has finished running.",
+ }
+ ActionCable.server.broadcast("notifications.#{user_id}", out)
end
+ end
- def fetch_docs_from_dataset(tool_id, experiment_id, user_id, dataset_id)
- d = Dataset.find(dataset_id)
- all_docs = []
- docs = []
- page = 0
- while page == 0 or docs.size == 100
- page += 1
- docs = d.fetch_paginated_documents(page, 100, "default", "asc", "all")[:docs]
- all_docs.concat docs
- out = {
- type: "completion_rate",
- tool_id: tool_id,
- experiment_id: experiment_id,
- completion: d.documents.size == 0 ? 0 : ((all_docs.size.to_f/d.documents.size)*100).to_i
- }
- ActionCable.server.broadcast("notifications.#{user_id}", out)
- end
- all_docs.map do |doc|
- {
+ def fetch_docs_from_dataset(tool_id, experiment_id, user_id, dataset_id)
+ d = Dataset.find(dataset_id)
+ all_docs = []
+ docs = []
+ page = 0
+ while page == 0 or docs.size == 100
+ page += 1
+ docs = d.fetch_paginated_documents(page, 100, "default", "asc", "all")[:docs]
+ all_docs.concat docs
+ out = {
+ type: "completion_rate",
+ tool_id: tool_id,
+ experiment_id: experiment_id,
+ completion: d.documents.size == 0 ? 0 : ((all_docs.size.to_f / d.documents.size) * 100).to_i,
+ }
+ ActionCable.server.broadcast("notifications.#{user_id}", out)
+ end
+ all_docs.map do |doc|
+ {
id: doc.id,
newspaper: doc.newspaper,
language: doc.language,
text: doc.all_text,
- date: doc.date_created
+ date: doc.date_created,
}
- end
end
+ end
end
diff --git a/app/workers/splitter_worker.rb b/app/workers/splitter_worker.rb
index bb90f61f40c6f4dbdd5f4ca652a99612b6672a1f..a6444d969eecde7f5a456b18052cb6f9a4513bd4 100644
--- a/app/workers/splitter_worker.rb
+++ b/app/workers/splitter_worker.rb
@@ -1,8 +1,6 @@
class SplitterWorker
- include Sidekiq::Worker
-
- def perform(tool_id, user_id, experiment_id, tool_type, tool_parameters)
-
- end
+ include Sidekiq::Worker
+ def perform(tool_id, user_id, experiment_id, tool_type, tool_parameters)
+ end
end
diff --git a/config/application.rb b/config/application.rb
index 27558195263f4adcf130c9bf0038b799aaaf3dd8..7eaa9a2cc47f649a7c8e3951ff27ceb45c8272bf 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -7,22 +7,33 @@ require "rails/all"
Bundler.require(*Rails.groups)
module NewspapersPlatform
- class Application < Rails::Application
- # Initialize configuration defaults for originally generated Rails version.
- config.load_defaults 6.1
+ class Application < Rails::Application
+ # Initialize configuration defaults for originally generated Rails version.
+ config.load_defaults 6.1
- # Configuration for the application, engines, and railties goes here.
- #
- # These settings can be overridden in specific environments using the files
- # in config/environments, which are processed later.
- #
- # config.time_zone = "Central Time (US & Canada)"
- # config.eager_load_paths << Rails.root.join("extras")
- config.solr = config_for('solr')
- if File.exist? "#{Rails.root}/config/auths.yml"
- config.auths = config_for('auths')
- end
- # config.active_job.queue_adapter = :sidekiq
- config.iiif_sources = config_for("iiif_sources")
+ # Configuration for the application, engines, and railties goes here.
+ #
+ # These settings can be overridden in specific environments using the files
+ # in config/environments, which are processed later.
+ #
+ # config.time_zone = "Central Time (US & Canada)"
+ # config.eager_load_paths << Rails.root.join("extras")
+ config.solr = config_for("solr")
+ if File.exist? "#{Rails.root}/config/auths.yml"
+ config.auths = config_for("auths")
end
+ # config.active_job.queue_adapter = :sidekiq
+ config.iiif_sources = config_for("iiif_sources")
+
+ # Allow requests from different origins
+ config.middleware.use Rack::Cors do
+ allow do
+ origins "*"
+ resource "*",
+ headers: :any,
+ expose: %w(access-token expiry token-type uid client),
+ methods: %i(post)
+ end
+ end
+ end
end
diff --git a/config/boot.rb b/config/boot.rb
index 3cda23b4db46f4304ea7df756bcefdf304bcd055..988a5ddc460f03c2cafb3b91d98f0cc930ebc8ed 100644
--- a/config/boot.rb
+++ b/config/boot.rb
@@ -1,4 +1,4 @@
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
require "bundler/setup" # Set up gems listed in the Gemfile.
require "bootsnap/setup" # Speed up boot time by caching expensive operations.
diff --git a/config/environments/production.rb b/config/environments/production.rb
index 2a69174e9956abf725760b632992f09133b9868f..b67d74104fa16590f31855168bdcbe4bb74d1fea 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -13,7 +13,7 @@ Rails.application.configure do
config.eager_load = true
# Full error reports are disabled and caching is turned on.
- config.consider_all_requests_local = false
+ config.consider_all_requests_local = false
config.action_controller.perform_caching = true
# Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"]
@@ -22,7 +22,7 @@ Rails.application.configure do
# Disable serving static files from the `/public` folder by default since
# Apache or NGINX already handles this.
- config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present?
+ config.public_file_server.enabled = ENV["RAILS_SERVE_STATIC_FILES"].present?
# Compress CSS using a preprocessor.
# config.assets.css_compressor = :sass
@@ -53,7 +53,7 @@ Rails.application.configure do
config.log_level = :info
# Prepend all log lines with the following tags.
- config.log_tags = [ :request_id ]
+ config.log_tags = [:request_id]
# Use a different cache store in production.
# config.cache_store = :mem_cache_store
@@ -89,9 +89,9 @@ Rails.application.configure do
# config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
if ENV["RAILS_LOG_TO_STDOUT"].present?
- logger = ActiveSupport::Logger.new(STDOUT)
+ logger = ActiveSupport::Logger.new(STDOUT)
logger.formatter = config.log_formatter
- config.logger = ActiveSupport::TaggedLogging.new(logger)
+ config.logger = ActiveSupport::TaggedLogging.new(logger)
end
# Do not dump schema after migrations.
diff --git a/config/environments/test.rb b/config/environments/test.rb
index 93ed4f1b78622a525958f919962735106b03152e..4fbe458abe9135b9463739cff523f19d4ff643d7 100644
--- a/config/environments/test.rb
+++ b/config/environments/test.rb
@@ -19,11 +19,11 @@ Rails.application.configure do
# Configure public file server for tests with Cache-Control for performance.
config.public_file_server.enabled = true
config.public_file_server.headers = {
- 'Cache-Control' => "public, max-age=#{1.hour.to_i}"
+ "Cache-Control" => "public, max-age=#{1.hour.to_i}",
}
# Show full error reports and disable caching.
- config.consider_all_requests_local = true
+ config.consider_all_requests_local = true
config.action_controller.perform_caching = false
config.cache_store = :null_store
diff --git a/config/initializers/assets.rb b/config/initializers/assets.rb
index 4b828e80cb778244843194acf5f8a74238ec2c13..c1f948d0184dfaad4f51fb3bd04802046aad1afc 100644
--- a/config/initializers/assets.rb
+++ b/config/initializers/assets.rb
@@ -1,12 +1,12 @@
# Be sure to restart your server when you modify this file.
# Version of your assets, change this if you want to expire all your assets.
-Rails.application.config.assets.version = '1.0'
+Rails.application.config.assets.version = "1.0"
# Add additional assets to the asset load path.
# Rails.application.config.assets.paths << Emoji.images_path
# Add Yarn node_modules folder to the asset load path.
-Rails.application.config.assets.paths << Rails.root.join('node_modules')
+Rails.application.config.assets.paths << Rails.root.join("node_modules")
# Precompile additional assets.
# application.js, application.css, and all non-JS/CSS in the app/assets
diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb
index 4b34a036689c21f4ccddb82823b2c2de065bb7fc..0af68a68b276c2ae8cff41ebae6908372594a1ec 100644
--- a/config/initializers/filter_parameter_logging.rb
+++ b/config/initializers/filter_parameter_logging.rb
@@ -2,5 +2,5 @@
# Configure sensitive parameters which will be filtered from the log file.
Rails.application.config.filter_parameters += [
- :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn
+ :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn,
]
diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb
index df3891e30f03bb67c0beabb4485d04d017db5c85..38f5f6928d320bd9e70c5d0a0a3b60ca337b0ad6 100644
--- a/config/initializers/session_store.rb
+++ b/config/initializers/session_store.rb
@@ -1,2 +1,2 @@
-Rails.application.config.session_store :active_record_store, :key => '_web_session'
-# Run this regularly to clean up DB: 'rails db:sessions:trim'
\ No newline at end of file
+Rails.application.config.session_store :active_record_store, :key => "_web_session"
+# Run this regularly to clean up DB: 'rails db:sessions:trim'
diff --git a/config/routes.rb b/config/routes.rb
index b753e954f0d2627604c85a8ffaee77f0e79a56bf..7ee124b150b46148416bef3989708ea143304970 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,58 +1,58 @@
# frozen_string_literal: true
-require 'sidekiq/web'
+require "sidekiq/web"
Rails.application.routes.draw do
- root to: 'catalog#home'
+ root to: "catalog#home"
- get '/send', to: "application#send_file"
+ get "/send", to: "application#send_file"
- get '/search', to: 'catalog#index'
- get '/catalog/:id', to: 'catalog#show'
- post '/catalog/facet_pagination', to: 'catalog#paginate_facets'
- post '/named_entities', to: 'catalog#named_entities_for_docs'
- post '/dataset_named_entities', to: 'catalog#named_entities_for_dataset'
- post '/catalog/wide_dates_histogram', to: 'catalog#wide_dates_histogram'
- post '/catalog/confirm_compound_creation', to: 'catalog#confirm_compound_creation'
- post '/catalog/create_compound', to: 'catalog#create_compound'
- post '/catalog/delete_compound', to: 'catalog#delete_compound'
- post '/catalog/random_sample', to: 'catalog#random_sample'
+ get "/search", to: "catalog#index"
+ get "/catalog/:id", to: "catalog#show"
+ post "/catalog/facet_pagination", to: "catalog#paginate_facets"
+ post "/named_entities", to: "catalog#named_entities_for_docs"
+ post "/dataset_named_entities", to: "catalog#named_entities_for_dataset"
+ post "/catalog/wide_dates_histogram", to: "catalog#wide_dates_histogram"
+ post "/catalog/confirm_compound_creation", to: "catalog#confirm_compound_creation"
+ post "/catalog/create_compound", to: "catalog#create_compound"
+ post "/catalog/delete_compound", to: "catalog#delete_compound"
+ post "/catalog/random_sample", to: "catalog#random_sample"
- get '/datasets', to: 'dataset#index'
- get '/datasets/update', to: 'dataset#update_datasets_list'
- post '/datasets/working_dataset', to: 'dataset#set_working_dataset'
- post "/datasets/add_selected_documents", to: "dataset#add_selected_documents"
- post "/datasets/add_compound", to: "dataset#add_compound"
- post "/datasets/remove_selected_documents", to: "dataset#remove_selected_documents"
- post "/datasets/add_all_documents", to: "dataset#add_all_documents"
- post "/datasets/export_dataset", to: "dataset#export_dataset"
- get "/datasets/list", to: "dataset#list_datasets"
- get '/dataset/:id', to: 'dataset#show'
- post "/dataset/toggle_sharing_status", to: "dataset#toggle_sharing_status"
- post "/dataset/:id/paginate", to: "dataset#paginate"
- post '/dataset/create', to: 'dataset#create_dataset'
- post '/dataset/rename', to: 'dataset#rename_dataset'
- post '/dataset/import', to: 'dataset#import_dataset'
- post '/dataset/delete', to: 'dataset#delete_dataset'
+ get "/datasets", to: "dataset#index"
+ get "/datasets/update", to: "dataset#update_datasets_list"
+ post "/datasets/working_dataset", to: "dataset#set_working_dataset"
+ post "/datasets/add_selected_documents", to: "dataset#add_selected_documents"
+ post "/datasets/add_compound", to: "dataset#add_compound"
+ post "/datasets/remove_selected_documents", to: "dataset#remove_selected_documents"
+ post "/datasets/add_all_documents", to: "dataset#add_all_documents"
+ post "/datasets/export_dataset", to: "dataset#export_dataset"
+ get "/datasets/list", to: "dataset#list_datasets"
+ get "/dataset/:id", to: "dataset#show"
+ post "/dataset/toggle_sharing_status", to: "dataset#toggle_sharing_status"
+ post "/dataset/:id/paginate", to: "dataset#paginate"
+ post "/dataset/create", to: "dataset#create_dataset"
+ post "/dataset/rename", to: "dataset#rename_dataset"
+ post "/dataset/import", to: "dataset#import_dataset"
+ post "/dataset/delete", to: "dataset#delete_dataset"
- get '/experiments', to: 'experiment#index'
- get '/experiments/update', to: 'experiment#update_experiments_list'
- post '/experiment/create', to: 'experiment#create'
- post '/experiment/delete', to: 'experiment#delete'
- post '/experiment/run', to: 'experiment#run_experiment'
- get '/experiment/:id', to: "experiment#show"
- get '/experiment/:id/load', to: "experiment#load"
- post '/experiment/:id/save', to: "experiment#save"
- post '/experiment/:id/add_tool', to: "experiment#add_tool"
- post '/experiment/:id/delete_tool', to: "experiment#delete_tool"
- post '/experiment/:id/edit_tool', to: "experiment#edit_tool"
- post '/experiment/:id/edit_tool_form', to: "experiment#edit_tool_form"
- post '/experiment/:id/tool_results', to: "experiment#tool_results"
- post '/experiment/:id/run_tool', to: "experiment#run_tool"
- post '/experiment/:id/run_experiment', to: "experiment#run_experiment"
+ get "/experiments", to: "experiment#index"
+ get "/experiments/update", to: "experiment#update_experiments_list"
+ post "/experiment/create", to: "experiment#create"
+ post "/experiment/delete", to: "experiment#delete"
+ post "/experiment/run", to: "experiment#run_experiment"
+ get "/experiment/:id", to: "experiment#show"
+ get "/experiment/:id/load", to: "experiment#load"
+ post "/experiment/:id/save", to: "experiment#save"
+ post "/experiment/:id/add_tool", to: "experiment#add_tool"
+ post "/experiment/:id/delete_tool", to: "experiment#delete_tool"
+ post "/experiment/:id/edit_tool", to: "experiment#edit_tool"
+ post "/experiment/:id/edit_tool_form", to: "experiment#edit_tool_form"
+ post "/experiment/:id/tool_results", to: "experiment#tool_results"
+ post "/experiment/:id/run_tool", to: "experiment#run_tool"
+ post "/experiment/:id/run_experiment", to: "experiment#run_experiment"
- resources :tool, only: [:show, :create, :update, :destroy]
+ resources :tool, only: [:show, :create, :update, :destroy]
- mount ActionCable.server => '/cable'
- if Rails.env.development?
- mount Sidekiq::Web => '/sidekiq'
- end
+ mount ActionCable.server => "/cable"
+ if Rails.env.development?
+ mount Sidekiq::Web => "/sidekiq"
+ end
end
diff --git a/db/migrate/20210721081005_create_users.rb b/db/migrate/20210721081005_create_users.rb
index 46e5b178dbe4817f5db36141574fbc7ccefdf04d..d2be7d5092714895ac83afaa2014b7cae8e91c5e 100644
--- a/db/migrate/20210721081005_create_users.rb
+++ b/db/migrate/20210721081005_create_users.rb
@@ -2,7 +2,7 @@ class CreateUsers < ActiveRecord::Migration[6.1]
def change
create_table :users do |t|
t.string :labs_user_id, null: false, unique: true
-
+
t.timestamps null: false
end
diff --git a/db/migrate/20210903194218_create_experiment.rb b/db/migrate/20210903194218_create_experiment.rb
index 8c0761e3ecd7f5342a104fc383aa0912e81df7c9..3dc1d36c18d132931feea4de8ce45e96f5ef4fab 100644
--- a/db/migrate/20210903194218_create_experiment.rb
+++ b/db/migrate/20210903194218_create_experiment.rb
@@ -1,11 +1,11 @@
class CreateExperiment < ActiveRecord::Migration[6.1]
- def change
- create_table :experiments do |t|
- t.string :title
- t.references :user, foreign_key: true
- t.jsonb :description, default: {children:[]}
- t.timestamps
- end
- add_index :experiments, [:title, :user_id], unique: true
+ def change
+ create_table :experiments do |t|
+ t.string :title
+ t.references :user, foreign_key: true
+ t.jsonb :description, default: { children: [] }
+ t.timestamps
end
+ add_index :experiments, [:title, :user_id], unique: true
+ end
end
diff --git a/db/migrate/20210909142841_create_datasets.rb b/db/migrate/20210909142841_create_datasets.rb
index 2e54da39e1a46a9cd478cb3d5b56d68ff75bb6df..44dfd60da850c13425606b1c900b4c44da8fb700 100644
--- a/db/migrate/20210909142841_create_datasets.rb
+++ b/db/migrate/20210909142841_create_datasets.rb
@@ -1,12 +1,12 @@
class CreateDatasets < ActiveRecord::Migration[6.1]
- def change
- create_table :datasets do |t|
- t.string :title
- t.references :user, foreign_key: true
- t.jsonb :documents, null: false, default: []
- t.boolean :public, default: false
- t.timestamps
- end
- add_index :datasets, [:title, :user_id], unique: true
+ def change
+ create_table :datasets do |t|
+ t.string :title
+ t.references :user, foreign_key: true
+ t.jsonb :documents, null: false, default: []
+ t.boolean :public, default: false
+ t.timestamps
end
+ add_index :datasets, [:title, :user_id], unique: true
+ end
end
diff --git a/db/migrate/20210915140752_add_tool.rb b/db/migrate/20210915140752_add_tool.rb
index b0363d37f12af95a8444c453bd1e47b7b75ba01a..2705fb61ace31e76bc139eed4b4b25ff5b586ee9 100644
--- a/db/migrate/20210915140752_add_tool.rb
+++ b/db/migrate/20210915140752_add_tool.rb
@@ -1,15 +1,15 @@
class AddTool < ActiveRecord::Migration[6.1]
def change
- create_table :tools do |t|
- t.references :experiment, foreign_key: true
- t.references :parent, foreign_key: {to_table: :tools}
- t.string :tool_type
- t.string :input_type
- t.string :output_type
- t.jsonb :parameters, default: {}
- t.jsonb :results, default: {}
- t.string :status, default: "created"
- t.timestamps
- end
+ create_table :tools do |t|
+ t.references :experiment, foreign_key: true
+ t.references :parent, foreign_key: { to_table: :tools }
+ t.string :tool_type
+ t.string :input_type
+ t.string :output_type
+ t.jsonb :parameters, default: {}
+ t.jsonb :results, default: {}
+ t.string :status, default: "created"
+ t.timestamps
+ end
end
end
diff --git a/db/migrate/20211110092535_add_notifications_table.rb b/db/migrate/20211110092535_add_notifications_table.rb
index 2b06ba89ee79db58fffcdf817a7f43b287c0baa6..9be63786e87014ca0b4f192c1545728041fb19ee 100644
--- a/db/migrate/20211110092535_add_notifications_table.rb
+++ b/db/migrate/20211110092535_add_notifications_table.rb
@@ -1,10 +1,10 @@
class AddNotificationsTable < ActiveRecord::Migration[6.1]
def change
- create_table :notifications do |t|
- t.references :user, foreign_key: true
- t.string :content
- t.boolean :read, default: false
- t.timestamps
- end
+ create_table :notifications do |t|
+ t.references :user, foreign_key: true
+ t.string :content
+ t.boolean :read, default: false
+ t.timestamps
+ end
end
end
diff --git a/db/migrate/20211123112405_create_compound_articles.rb b/db/migrate/20211123112405_create_compound_articles.rb
index 53bc24fc2ef5fc07aceab6981ada783f07ad7cc8..3b34a12159ce04d1b68f04fb99a2d7326e92dcc2 100644
--- a/db/migrate/20211123112405_create_compound_articles.rb
+++ b/db/migrate/20211123112405_create_compound_articles.rb
@@ -1,17 +1,17 @@
class CreateCompoundArticles < ActiveRecord::Migration[6.1]
- def change
- create_table :compound_articles do |t|
- t.string :title
- t.string :issue_id
- t.string :newspaper
- t.string :date_created
- t.string :thumbnail_url
- t.string :language
- t.text :all_text
- t.references :user, foreign_key: true
- t.string :parts, array: true, default: []
- t.timestamps
- end
- add_index :compound_articles, [:title, :user_id], unique: true
+ def change
+ create_table :compound_articles do |t|
+ t.string :title
+ t.string :issue_id
+ t.string :newspaper
+ t.string :date_created
+ t.string :thumbnail_url
+ t.string :language
+ t.text :all_text
+ t.references :user, foreign_key: true
+ t.string :parts, array: true, default: []
+ t.timestamps
end
+ add_index :compound_articles, [:title, :user_id], unique: true
+ end
end
diff --git a/db/schema.rb b/db/schema.rb
index 164227fa5c68f127839130f3eb05d190a9f22819..bc77a9743cb6ef840820d50377795764c21df84e 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -45,7 +45,7 @@ ActiveRecord::Schema.define(version: 2021_12_02_095539) do
create_table "experiments", force: :cascade do |t|
t.string "title"
t.bigint "user_id"
- t.jsonb "description", default: {"children"=>[]}
+ t.jsonb "description", default: { "children" => [] }
t.datetime "created_at", precision: 6, null: false
t.datetime "updated_at", precision: 6, null: false
t.index ["title", "user_id"], name: "index_experiments_on_title_and_user_id", unique: true
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 47b598dee48a9c7280887f4f5a86339f39b02351..d713e377c9488bfa4cfc9e4a5ed03146e42988e0 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -1,4 +1,4 @@
-ENV['RAILS_ENV'] ||= 'test'
+ENV["RAILS_ENV"] ||= "test"
require_relative "../config/environment"
require "rails/test_help"
diff --git a/test/workers/tool_runner_worker_test.rb b/test/workers/tool_runner_worker_test.rb
index 29db595b6f15a2c8dfe004c73b50df039d4f577c..95a67b1d902aabc4d8d63dd9941ec5e83b74a375 100644
--- a/test/workers/tool_runner_worker_test.rb
+++ b/test/workers/tool_runner_worker_test.rb
@@ -1,4 +1,5 @@
-require 'test_helper'
+require "test_helper"
+
class ToolRunnerWorkerTest < Minitest::Test
def test_example
skip "add some examples to (or delete) #{__FILE__}"