Skip to content
Commits on Source (39)
......@@ -41,3 +41,4 @@ yarn-debug.log*
/.idea/*
/config/auths.yml
db_env_var.txt
source 'https://rubygems.org'
source "https://rubygems.org"
git_source(:github) { |repo| "https://github.com/#{repo}.git" }
ruby '3.0.1'
ruby "3.0.1"
# Bundle edge Rails instead: gem 'rails', github: 'rails/rails', branch: 'main'
gem 'rails', '~> 6.1.3', '>= 6.1.3.2'
gem "rails", "~> 6.1.3", ">= 6.1.3.2"
# Use sqlite3 as the database for Active Record
# gem 'sqlite3', '~> 1.4'
gem 'pg'
gem 'devise'
# gem 'sqlite3'
gem "pg"
# gem 'devise'
# Use Puma as the app server
gem 'puma', '~> 5.0'
gem "puma", "~> 5.0"
# Use SCSS for stylesheets
gem 'sass-rails', '>= 6'
gem "sass-rails", ">= 6"
# Transpile app-like JavaScript. Read more: https://github.com/rails/webpacker
gem 'webpacker', '~> 5.0'
gem "webpacker"
# Turbolinks makes navigating your web application faster. Read more: https://github.com/turbolinks/turbolinks
gem 'turbolinks', '~> 5'
gem "turbolinks", "~> 5"
# Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
gem 'jbuilder', '~> 2.7'
gem "jbuilder", "~> 2.7"
# Use Redis adapter to run Action Cable in production
# gem 'redis', '~> 4.0'
# Use Active Model has_secure_password
# gem 'bcrypt', '~> 3.1.7'
gem 'rsolr'
gem 'sidekiq', '~> 6.0'
gem 'pragmatic_tokenizer'
gem 'activerecord-session_store'
# Use OpenSSL for encryption of Auth Token
gem "openssl"
# Use rSolr for search queries
gem "rsolr"
# Use rack-cors for cross-origin requests
gem "rack-cors"
gem "sidekiq", "~> 6.0"
gem "pragmatic_tokenizer"
gem "activerecord-session_store"
# Use Active Storage variant
# gem 'image_processing', '~> 1.2'
# Reduces boot times through caching; required in config/boot.rb
gem 'bootsnap', '>= 1.4.4', require: false
gem "bootsnap", ">= 1.4.4", require: false
group :development, :test do
# Call 'byebug' anywhere in the code to stop execution and get a debugger console
gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
gem "byebug", platforms: [:mri, :mingw, :x64_mingw]
end
group :development do
# Access an interactive console on exception pages or by calling 'console' anywhere in the code.
gem 'web-console', '>= 4.1.0'
gem "web-console", ">= 4.1.0"
# Display performance information such as SQL time and flame graphs for each request in your browser.
# Can be configured to work on production as well see: https://github.com/MiniProfiler/rack-mini-profiler/blob/master/README.md
gem 'rack-mini-profiler', '~> 2.0'
gem 'listen', '~> 3.3'
gem "rack-mini-profiler", "~> 2.0"
gem "listen", "~> 3.3"
# Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
gem 'spring'
gem 'brakeman'
gem "spring"
gem "brakeman"
end
group :test do
# Adds support for Capybara system testing and selenium driver
gem 'capybara', '>= 3.26'
gem 'selenium-webdriver'
gem "capybara", ">= 3.26"
gem "selenium-webdriver"
# Easy installation and use of web drivers to run system tests with browsers
gem 'webdrivers'
gem "webdrivers"
end
# Windows does not include zoneinfo files, so bundle the tzinfo-data gem
gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
gem "tzinfo-data", platforms: [:mingw, :mswin, :x64_mingw, :jruby]
GEM
remote: https://rubygems.org/
specs:
actioncable (6.1.4)
actionpack (= 6.1.4)
activesupport (= 6.1.4)
actioncable (6.1.6.1)
actionpack (= 6.1.6.1)
activesupport (= 6.1.6.1)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
actionmailbox (6.1.4)
actionpack (= 6.1.4)
activejob (= 6.1.4)
activerecord (= 6.1.4)
activestorage (= 6.1.4)
activesupport (= 6.1.4)
actionmailbox (6.1.6.1)
actionpack (= 6.1.6.1)
activejob (= 6.1.6.1)
activerecord (= 6.1.6.1)
activestorage (= 6.1.6.1)
activesupport (= 6.1.6.1)
mail (>= 2.7.1)
actionmailer (6.1.4)
actionpack (= 6.1.4)
actionview (= 6.1.4)
activejob (= 6.1.4)
activesupport (= 6.1.4)
actionmailer (6.1.6.1)
actionpack (= 6.1.6.1)
actionview (= 6.1.6.1)
activejob (= 6.1.6.1)
activesupport (= 6.1.6.1)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
actionpack (6.1.4)
actionview (= 6.1.4)
activesupport (= 6.1.4)
actionpack (6.1.6.1)
actionview (= 6.1.6.1)
activesupport (= 6.1.6.1)
rack (~> 2.0, >= 2.0.9)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.2.0)
actiontext (6.1.4)
actionpack (= 6.1.4)
activerecord (= 6.1.4)
activestorage (= 6.1.4)
activesupport (= 6.1.4)
actiontext (6.1.6.1)
actionpack (= 6.1.6.1)
activerecord (= 6.1.6.1)
activestorage (= 6.1.6.1)
activesupport (= 6.1.6.1)
nokogiri (>= 1.8.5)
actionview (6.1.4)
activesupport (= 6.1.4)
actionview (6.1.6.1)
activesupport (= 6.1.6.1)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.1, >= 1.2.0)
activejob (6.1.4)
activesupport (= 6.1.4)
activejob (6.1.6.1)
activesupport (= 6.1.6.1)
globalid (>= 0.3.6)
activemodel (6.1.4)
activesupport (= 6.1.4)
activerecord (6.1.4)
activemodel (= 6.1.4)
activesupport (= 6.1.4)
activemodel (6.1.6.1)
activesupport (= 6.1.6.1)
activerecord (6.1.6.1)
activemodel (= 6.1.6.1)
activesupport (= 6.1.6.1)
activerecord-session_store (2.0.0)
actionpack (>= 5.2.4.1)
activerecord (>= 5.2.4.1)
multi_json (~> 1.11, >= 1.11.2)
rack (>= 2.0.8, < 3)
railties (>= 5.2.4.1)
activestorage (6.1.4)
actionpack (= 6.1.4)
activejob (= 6.1.4)
activerecord (= 6.1.4)
activesupport (= 6.1.4)
marcel (~> 1.0.0)
activestorage (6.1.6.1)
actionpack (= 6.1.6.1)
activejob (= 6.1.6.1)
activerecord (= 6.1.6.1)
activesupport (= 6.1.6.1)
marcel (~> 1.0)
mini_mime (>= 1.1.0)
activesupport (6.1.4)
activesupport (6.1.6.1)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 1.6, < 2)
minitest (>= 5.1)
......@@ -68,124 +68,110 @@ GEM
zeitwerk (~> 2.3)
addressable (2.8.0)
public_suffix (>= 2.0.2, < 5.0)
bcrypt (3.1.16)
bindex (0.8.1)
bootsnap (1.7.5)
msgpack (~> 1.0)
brakeman (5.1.1)
bootsnap (1.13.0)
msgpack (~> 1.2)
brakeman (5.2.3)
builder (3.2.4)
byebug (11.1.3)
capybara (3.35.3)
capybara (3.37.1)
addressable
matrix
mini_mime (>= 0.1.3)
nokogiri (~> 1.8)
rack (>= 1.6.0)
rack-test (>= 0.6.3)
regexp_parser (>= 1.5, < 3.0)
xpath (~> 3.2)
childprocess (3.0.0)
concurrent-ruby (1.1.9)
childprocess (4.1.0)
concurrent-ruby (1.1.10)
connection_pool (2.2.5)
crass (1.0.6)
devise (4.8.0)
bcrypt (~> 3.0)
orm_adapter (~> 0.1)
railties (>= 4.1.0)
responders
warden (~> 1.2.3)
erubi (1.10.0)
faraday (1.4.2)
faraday-em_http (~> 1.0)
faraday-em_synchrony (~> 1.0)
faraday-excon (~> 1.1)
faraday-net_http (~> 1.0)
faraday-net_http_persistent (~> 1.1)
multipart-post (>= 1.2, < 3)
faraday (2.4.0)
faraday-net_http (~> 2.0)
ruby2_keywords (>= 0.0.4)
faraday-em_http (1.0.0)
faraday-em_synchrony (1.0.0)
faraday-excon (1.1.0)
faraday-net_http (1.0.1)
faraday-net_http_persistent (1.1.0)
ffi (1.15.3)
globalid (0.4.2)
activesupport (>= 4.2.0)
i18n (1.8.10)
faraday-net_http (2.1.0)
ffi (1.15.5)
globalid (1.0.0)
activesupport (>= 5.0)
i18n (1.12.0)
concurrent-ruby (~> 1.0)
jbuilder (2.11.2)
jbuilder (2.11.5)
actionview (>= 5.0.0)
activesupport (>= 5.0.0)
listen (3.6.0)
listen (3.7.1)
rb-fsevent (~> 0.10, >= 0.10.3)
rb-inotify (~> 0.9, >= 0.9.10)
loofah (2.10.0)
loofah (2.18.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
marcel (1.0.1)
marcel (1.0.2)
matrix (0.4.2)
method_source (1.0.0)
mini_mime (1.1.0)
minitest (5.14.4)
msgpack (1.4.2)
mini_mime (1.1.2)
minitest (5.16.2)
msgpack (1.5.4)
multi_json (1.15.0)
multipart-post (2.1.1)
nio4r (2.5.7)
nokogiri (1.11.7-x86_64-linux)
nio4r (2.5.8)
nokogiri (1.13.8-x86_64-linux)
racc (~> 1.4)
orm_adapter (0.5.0)
pg (1.2.3)
openssl (3.0.0)
pg (1.4.2)
pragmatic_tokenizer (3.2.0)
unicode
public_suffix (4.0.6)
puma (5.3.2)
public_suffix (4.0.7)
puma (5.6.4)
nio4r (~> 2.0)
racc (1.5.2)
rack (2.2.3)
rack-mini-profiler (2.3.2)
racc (1.6.0)
rack (2.2.4)
rack-cors (1.1.1)
rack (>= 2.0.0)
rack-mini-profiler (2.3.4)
rack (>= 1.2.0)
rack-proxy (0.7.0)
rack-proxy (0.7.2)
rack
rack-test (1.1.0)
rack (>= 1.0, < 3)
rails (6.1.4)
actioncable (= 6.1.4)
actionmailbox (= 6.1.4)
actionmailer (= 6.1.4)
actionpack (= 6.1.4)
actiontext (= 6.1.4)
actionview (= 6.1.4)
activejob (= 6.1.4)
activemodel (= 6.1.4)
activerecord (= 6.1.4)
activestorage (= 6.1.4)
activesupport (= 6.1.4)
rack-test (2.0.2)
rack (>= 1.3)
rails (6.1.6.1)
actioncable (= 6.1.6.1)
actionmailbox (= 6.1.6.1)
actionmailer (= 6.1.6.1)
actionpack (= 6.1.6.1)
actiontext (= 6.1.6.1)
actionview (= 6.1.6.1)
activejob (= 6.1.6.1)
activemodel (= 6.1.6.1)
activerecord (= 6.1.6.1)
activestorage (= 6.1.6.1)
activesupport (= 6.1.6.1)
bundler (>= 1.15.0)
railties (= 6.1.4)
railties (= 6.1.6.1)
sprockets-rails (>= 2.0.0)
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
rails-html-sanitizer (1.3.0)
rails-html-sanitizer (1.4.3)
loofah (~> 2.3)
railties (6.1.4)
actionpack (= 6.1.4)
activesupport (= 6.1.4)
railties (6.1.6.1)
actionpack (= 6.1.6.1)
activesupport (= 6.1.6.1)
method_source
rake (>= 0.13)
rake (>= 12.2)
thor (~> 1.0)
rake (13.0.6)
rb-fsevent (0.11.0)
rb-fsevent (0.11.1)
rb-inotify (0.10.1)
ffi (~> 1.0)
redis (4.4.0)
regexp_parser (2.1.1)
responders (3.0.1)
actionpack (>= 5.0)
railties (>= 5.0)
rsolr (2.3.0)
redis (4.7.1)
regexp_parser (2.5.0)
rexml (3.2.5)
rsolr (2.5.0)
builder (>= 2.1.2)
faraday (>= 0.9.0)
ruby2_keywords (0.0.4)
faraday (>= 0.9, < 3, != 2.0.0)
ruby2_keywords (0.0.5)
rubyzip (2.3.2)
sass-rails (6.0.0)
sassc-rails (~> 2.1, >= 2.1.1)
......@@ -197,52 +183,53 @@ GEM
sprockets (> 3.0)
sprockets-rails
tilt
selenium-webdriver (3.142.7)
childprocess (>= 0.5, < 4.0)
rubyzip (>= 1.2.2)
selenium-webdriver (4.3.0)
childprocess (>= 0.5, < 5.0)
rexml (~> 3.2, >= 3.2.5)
rubyzip (>= 1.2.2, < 3.0)
websocket (~> 1.0)
semantic_range (3.0.0)
sidekiq (6.2.2)
sidekiq (6.5.1)
connection_pool (>= 2.2.2)
rack (~> 2.0)
redis (>= 4.2.0)
spring (2.1.1)
sprockets (4.0.2)
spring (4.0.0)
sprockets (4.1.1)
concurrent-ruby (~> 1.0)
rack (> 1, < 3)
sprockets-rails (3.2.2)
actionpack (>= 4.0)
activesupport (>= 4.0)
sprockets-rails (3.4.2)
actionpack (>= 5.2)
activesupport (>= 5.2)
sprockets (>= 3.0.0)
thor (1.1.0)
tilt (2.0.10)
thor (1.2.1)
tilt (2.0.11)
turbolinks (5.2.1)
turbolinks-source (~> 5.2)
turbolinks-source (5.2.0)
tzinfo (2.0.4)
tzinfo (2.0.5)
concurrent-ruby (~> 1.0)
unicode (0.4.4.4)
warden (1.2.9)
rack (>= 2.0.9)
web-console (4.1.0)
web-console (4.2.0)
actionview (>= 6.0.0)
activemodel (>= 6.0.0)
bindex (>= 0.4.0)
railties (>= 6.0.0)
webdrivers (4.6.0)
webdrivers (5.0.0)
nokogiri (~> 1.6)
rubyzip (>= 1.3.0)
selenium-webdriver (>= 3.0, < 4.0)
webpacker (5.4.0)
selenium-webdriver (~> 4.0)
webpacker (5.4.3)
activesupport (>= 5.2)
rack-proxy (>= 0.6.1)
railties (>= 5.2)
semantic_range (>= 2.3.0)
websocket (1.2.9)
websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
xpath (3.2.0)
nokogiri (~> 1.8)
zeitwerk (2.4.2)
zeitwerk (2.6.0)
PLATFORMS
x86_64-linux
......@@ -253,12 +240,13 @@ DEPENDENCIES
brakeman
byebug
capybara (>= 3.26)
devise
jbuilder (~> 2.7)
listen (~> 3.3)
openssl
pg
pragmatic_tokenizer
puma (~> 5.0)
rack-cors
rack-mini-profiler (~> 2.0)
rails (~> 6.1.3, >= 6.1.3.2)
rsolr
......@@ -270,7 +258,7 @@ DEPENDENCIES
tzinfo-data
web-console (>= 4.1.0)
webdrivers
webpacker (~> 5.0)
webpacker
RUBY VERSION
ruby 3.0.1p64
......
module ApplicationCable
##
# This class identifies the current user in a websocket communication using ApplicationCable
##
# This class identifies the current user in a websocket communication using ApplicationCable
class Connection < ActionCable::Connection::Base
identified_by :current_user
identified_by :current_user
def connect
self.current_user = find_user
end
def connect
self.current_user = find_user
end
def find_user
user_id = cookies.signed["user.id"]
current_user = User.find_by(id: user_id)
if current_user
current_user
else
reject_unauthorized_connection
end
def find_user
if current_user = User.find_by(id: cookies.encrypted['_web_session']['current_user_id'])
current_user
else
reject_unauthorized_connection
end
end
end
end
class ApplicationController < ActionController::Base
include Authentication
include AddProxyRequestOrigin
def send_file
File.open("tmp/#{params[:filename]}", 'r') do |f|
send_data f.read, type: "text/json", filename: params[:filename]
end
def send_file
File.open("tmp/#{params[:filename]}", "r") do |f|
send_data f.read, type: "text/json", filename: params[:filename]
end
end
end
class CatalogController < ApplicationController
before_action :authenticate_user!, :strip_input_fields
before_action :authenticate_user!, :strip_input_fields
def home
end
def home
end
##
# Creates a search query and submit it to the index. Retrieve and displays results + metadata.
def index
if params[:q]
@search_type = params[:search_type].nil? ? "exact" : params[:search_type]
@solr_params = SolrQuery.new(@search_type).to_params
@solr_params[:q] = params[:q]
@solr_params[:rows] = params[:per_page] if params[:per_page]
@current_page = params[:page].to_i != 0 ? params[:page].to_i : 1
@solr_params[:start] = params[:page].to_i != 0 ? @solr_params[:rows] * (params[:page].to_i-1) : 0
@solr_params[:sort] = params[:sort] if params[:sort]
if params[:f]
params[:f].each do |k,v|
if k == "date_created_dtsi" # v is a hash {to: "", from: ""}
@solr_params[:fq] << "#{k}:[#{v['from']}T00:00:00Z TO #{v['to']}T00:00:00Z]"
else
if v.is_a? Array
v.each do |val|
@solr_params[:fq] << "#{k}:#{val}"
end
end
end
end
##
# Creates a search query and submit it to the index. Retrieve and displays results + metadata.
def index
if params[:q]
@search_type = params[:search_type].nil? ? "exact" : params[:search_type]
@solr_params = SolrQuery.new(@search_type).to_params
@solr_params[:q] = params[:q]
@solr_params[:rows] = params[:per_page] if params[:per_page]
@current_page = params[:page].to_i != 0 ? params[:page].to_i : 1
@solr_params[:start] = params[:page].to_i != 0 ? @solr_params[:rows] * (params[:page].to_i - 1) : 0
@solr_params[:sort] = params[:sort] if params[:sort]
if params[:f]
params[:f].each do |k, v|
if k == "date_created_dtsi" # v is a hash {to: "", from: ""}
@solr_params[:fq] << "#{k}:[#{v["from"]}T00:00:00Z TO #{v["to"]}T00:00:00Z]"
else
if v.is_a? Array
v.each do |val|
@solr_params[:fq] << "#{k}:#{val}"
end
end
session['search_params'] = @solr_params
session['query_params'] = params.to_unsafe_h.slice('q', 'page', 'per_page','sort', 'f')
@results = SolrSearcher.query @solr_params
puts @results.to_json if Rails.env == "development"
@resulting_docs = @results['response']['docs'].map do |solr_doc|
case solr_doc['has_model_ssim']
when ['Article']
Article.from_solr_doc solr_doc
when ['Issue']
Issue.from_solr_doc solr_doc
end
end
entities_fields = I18n.t("newspapers.solr_fields").values_at(:persons, :locations, :organisations, :human_productions)
@entities_labels = []
entities_fields.each do |entity_field|
(@entities_labels << @results['facets'][entity_field]['buckets'].map{|ne| ne['val']}).flatten! if @results['facets'][entity_field]
end
@entities_labels = helpers.get_entity_label @entities_labels
end
end
end
session["search_params"] = @solr_params
session["query_params"] = params.to_unsafe_h.slice("q", "page", "per_page", "sort", "f")
@results = SolrSearcher.query @solr_params
# puts @results.to_json if Rails.env == "development"
@resulting_docs = @results["response"]["docs"].map do |solr_doc|
case solr_doc["has_model_ssim"]
when ["Article"]
Article.from_solr_doc solr_doc
when ["Issue"]
Issue.from_solr_doc solr_doc
end
end
entities_fields = I18n.t("newspapers.solr_fields").values_at(:persons, :locations, :organisations, :human_productions)
@entities_labels = []
entities_fields.each do |entity_field|
(@entities_labels << @results["facets"][entity_field]["buckets"].map { |ne| ne["val"] }).flatten! if @results["facets"][entity_field]
end
@entities_labels = helpers.get_entity_label @entities_labels
end
end
##
# Display an issue
def show
@issue = Issue.from_solr params[:id], with_pages=true, with_articles=true
session['named_entities'] = Issue.named_entities @issue.id
session['named_entities_labels'] = helpers.get_linked_entities session['named_entities'].map{ |k,v| v.keys }.flatten.uniq
end
##
# Display an issue
def show
@issue = Issue.from_solr params[:id], with_pages = true, with_articles = true
end
##
# Retrieve named entities for a list of documents (issue and/or articles)
def named_entities_for_docs
named_entities = {LOC: {}, PER: {}, ORG: {}, HumanProd: {}}
params[:docs_ids].each do |doc_id|
if doc_id.index('_article_').nil?
doc_named_entities = session['named_entities']
else # if article, filter stored list
doc_named_entities = session['named_entities'].map{ |ne_type, ne_list|
[ne_type,ne_list.select{ |linked_id, namedentities|
namedentities.any?{ |ne|
ne['article_id_ssi'] == doc_id
}
}.map{ |k,v| [k,v.select{ |ne| ne['article_id_ssi'] == doc_id }] }.to_h]
}.to_h
end
named_entities[:LOC] = named_entities[:LOC].merge(doc_named_entities[:LOC]) do |key,oldval,newval|
oldval.concat newval
end
named_entities[:ORG] = named_entities[:ORG].merge(doc_named_entities[:ORG]) do |key,oldval,newval|
oldval.concat newval
end
named_entities[:PER] = named_entities[:PER].merge(doc_named_entities[:PER]) do |key,oldval,newval|
oldval.concat newval
end
named_entities[:HumanProd] = named_entities[:HumanProd].merge(doc_named_entities[:HumanProd]) do |key,oldval,newval|
oldval.concat newval
end
end
render partial: 'named_entities/named_entities', locals: {named_entities: named_entities, linked_entities: session['named_entities_labels']}
##
# Retrieve named entities for a list of documents (issue and/or articles)
def named_entities_for_docs
named_entities = { LOC: {}, PER: {}, ORG: {}, HumanProd: {} }
linked_entities = {}
params[:docs_ids].each do |doc_id|
if doc_id.index("_article_").nil? # if issue, get named entities from Issue
doc_named_entities = Issue.named_entities doc_id
else # if article, filter stored list
issue_id = doc_id.split(/_article_/, 2).first
doc_named_entities = Issue.named_entities issue_id
doc_named_entities = doc_named_entities.map { |ne_type, ne_list|
[ne_type, ne_list.select { |linked_id, namedentities|
namedentities.any? { |ne|
ne["article_id_ssi"] == doc_id
}
}.map { |k, v| [k, v.select { |ne| ne["article_id_ssi"] == doc_id }] }.to_h]
}.to_h
end
named_entities[:LOC] = named_entities[:LOC].merge(doc_named_entities[:LOC]) do |key, oldval, newval|
oldval.concat newval
end
named_entities[:ORG] = named_entities[:ORG].merge(doc_named_entities[:ORG]) do |key, oldval, newval|
oldval.concat newval
end
named_entities[:PER] = named_entities[:PER].merge(doc_named_entities[:PER]) do |key, oldval, newval|
oldval.concat newval
end
named_entities[:HumanProd] = named_entities[:HumanProd].merge(doc_named_entities[:HumanProd]) do |key, oldval, newval|
oldval.concat newval
end
doc_named_entities_labels = helpers.get_linked_entities doc_named_entities.map { |k, v| v.keys }.flatten.uniq
linked_entities.merge!(doc_named_entities_labels)
end
render partial: "named_entities/named_entities", locals: { named_entities: named_entities, linked_entities: linked_entities }
end
##
# Retrieve named entities for a dataset
def named_entities_for_dataset
dataset = Dataset.find(params[:dataset_id])
named_entities = dataset.named_entities
named_entities_labels = helpers.get_linked_entities named_entities.map{ |k,v| v.keys }.flatten.uniq
render partial: 'named_entities/named_entities', locals: {named_entities: named_entities, linked_entities: named_entities_labels}
end
##
# Retrieve named entities for a dataset
def named_entities_for_dataset
dataset = Dataset.find(params[:dataset_id])
named_entities = dataset.named_entities
named_entities_labels = helpers.get_linked_entities named_entities.map { |k, v| v.keys }.flatten.uniq
render partial: "named_entities/named_entities", locals: { named_entities: named_entities, linked_entities: named_entities_labels }
end
##
# Retrieve and display paginated facets
def paginate_facets
out = {}
if params[:field_name] != ""
search_params = session['search_params']
search_params['rows'] = 0
search_params['json.facet'] = {"#{params[:field_name]}": {terms: {
field: params[:field_name],
limit: 15,
numBuckets: true,
offset: (params[:current_page].to_i-1) * 15}}}.to_json
res = SolrSearcher.query search_params
entities_labels = [res['facets'][params[:field_name]]['buckets'].map{|ne| ne['val']}]
entities_labels = helpers.get_entity_label entities_labels
facet_constraints = search_params['fq'].select { |fq| fq.split(':')[0] == params[:field_name] }.map{|fq| {label: params[:field_name], value: fq.split(':')[1]} }
out[:facets_entries] = []
res['facets'][params[:field_name]]['buckets'].each do |facet_entry|
out[:facets_entries] << render_to_string(layout: false, partial: "facet_entry", locals: {
entities_labels: entities_labels,
facet_constraints: facet_constraints,
field: params[:field_name],
facet: facet_entry,
index: params[:current_page].to_i,
per_page: 15
})
end
end
out[:pagination] = render_to_string(layout: false, partial: 'facet_pagination', locals: {nb_pages: params[:nb_pages].to_i, current_page: params[:current_page].to_i})
render json: out
##
# Retrieve and display paginated facets
def paginate_facets
out = {}
if params[:field_name] != ""
search_params = session["search_params"]
search_params["rows"] = 0
search_params["json.facet"] = { "#{params[:field_name]}": { terms: {
field: params[:field_name],
limit: 15,
numBuckets: true,
offset: (params[:current_page].to_i - 1) * 15,
} } }.to_json
res = SolrSearcher.query search_params
entities_labels = [res["facets"][params[:field_name]]["buckets"].map { |ne| ne["val"] }]
entities_labels = helpers.get_entity_label entities_labels
facet_constraints = search_params["fq"].select { |fq| fq.split(":")[0] == params[:field_name] }.map { |fq| { label: params[:field_name], value: fq.split(":")[1] } }
out[:facets_entries] = []
res["facets"][params[:field_name]]["buckets"].each do |facet_entry|
out[:facets_entries] << render_to_string(layout: false, partial: "facet_entry", locals: {
entities_labels: entities_labels,
facet_constraints: facet_constraints,
field: params[:field_name],
facet: facet_entry,
index: params[:current_page].to_i,
per_page: 15,
})
end
end
out[:pagination] = render_to_string(layout: false, partial: "facet_pagination", locals: { nb_pages: params[:nb_pages].to_i, current_page: params[:current_page].to_i })
render json: out
end
##
# Open modal for date frequencies histogram in wide format
def wide_dates_histogram
out = {}
out[:modal_content] = render_to_string(layout: false, partial: "wide_dates_histogram")
render json: out
end
##
# Open modal for date frequencies histogram in wide format
def wide_dates_histogram
out = {}
out[:modal_content] = render_to_string(layout: false, partial: "wide_dates_histogram")
render json: out
end
##
# Open Modal to confirm the creation of a compound article
def confirm_compound_creation
out = {}
out[:modal_content] = render_to_string(layout: false, partial: "confirm_compound_creation", locals: {article_parts: params[:article_parts]})
render json: out
end
##
# Open Modal to confirm the creation of a compound article
def confirm_compound_creation
out = {}
out[:modal_content] = render_to_string(layout: false, partial: "confirm_compound_creation", locals: { article_parts: params[:article_parts] })
render json: out
end
##
# Create a new compound article
def create_compound
compound = CompoundArticle.new
compound.user = current_user
compound.title = params[:title]
compound.issue_id = params[:issue_id]
issue = Issue.from_solr params[:issue_id]
compound.newspaper = issue.newspaper
compound.date_created = issue.date_created
compound.thumbnail_url = issue.thumbnail_url
compound.language = issue.language
compound.all_text = params[:all_text]
compound.parts = params[:article_parts_ids]
begin
compound.save!
render json: {status: 'ok', html: render_to_string(layout: false, partial: "compound_articles_panel", locals: {issue_id: params[:issue_id]})}
rescue ActiveRecord::RecordNotUnique
render json: {status: "error", message: "A compound article with this title already exists."}
rescue ActiveRecord::RecordInvalid
render json: {status: "error", message: "The title should not be blank."}
end
##
# Create a new compound article
def create_compound
compound = CompoundArticle.new
compound.user = current_user
compound.title = params[:title]
compound.issue_id = params[:issue_id]
issue = Issue.from_solr params[:issue_id]
compound.newspaper = issue.newspaper
compound.date_created = issue.date_created
compound.thumbnail_url = issue.thumbnail_url
compound.language = issue.language
compound.all_text = params[:all_text]
compound.parts = params[:article_parts_ids]
begin
compound.save!
render json: { status: "ok", html: render_to_string(layout: false, partial: "compound_articles_panel", locals: { issue_id: params[:issue_id] }) }
rescue ActiveRecord::RecordNotUnique
render json: { status: "error", message: "A compound article with this title already exists." }
rescue ActiveRecord::RecordInvalid
render json: { status: "error", message: "The title should not be blank." }
end
end
##
# Delete an existing compound
def delete_compound
compound = CompoundArticle.find(params[:compound_id])
issue_id = compound.issue_id
current_user.datasets.each do |dataset|
if dataset.documents.any?{|doc| doc['id'].to_s == compound.id.to_s}
dataset.documents = dataset.documents.select{|doc| doc['id'].to_s != compound.id.to_s}
dataset.save!
end
end
compound.destroy
out = {}
out[:html] = render_to_string(layout: false, partial: "compound_articles_panel", locals: {issue_id: issue_id})
out[:datasets] = render_to_string(layout: false, partial: "manage_datasets_content_show_page")
render json: out
##
# Delete an existing compound
def delete_compound
compound = CompoundArticle.find(params[:compound_id])
issue_id = compound.issue_id
current_user.datasets.each do |dataset|
if dataset.documents.any? { |doc| doc["id"].to_s == compound.id.to_s }
dataset.documents = dataset.documents.select { |doc| doc["id"].to_s != compound.id.to_s }
dataset.save!
end
end
compound.destroy
out = {}
out[:html] = render_to_string(layout: false, partial: "compound_articles_panel", locals: { issue_id: issue_id })
out[:datasets] = render_to_string(layout: false, partial: "manage_datasets_content_show_page")
render json: out
end
##
# Retrieve and display a random sample of the result of a search
def random_sample
search_params = session['search_params'].with_indifferent_access
search_params[:fq] = search_params[:fq].select {|elt| !elt.start_with? "has_model_ssim:" } if search_params[:fq]
search_params[:fq] ||= []
search_params[:fq] << "has_model_ssim:Article"
search_params[:sort] = "rand#{(0...8).map { (65 + rand(26)).chr }.join} asc"
results = SolrSearcher.query search_params
results = results['response']['docs'].map do |solr_doc|
case solr_doc['has_model_ssim']
when ['Article']
Article.from_solr_doc solr_doc
when ['Issue']
Issue.from_solr_doc solr_doc
end
end
render json: {content: render_to_string(layout: false, partial: "random_sample", locals: {resulting_docs: results}) }
##
# Retrieve and display a random sample of the result of a search
def random_sample
search_params = session["search_params"].with_indifferent_access
search_params[:fq] = search_params[:fq].select { |elt| !elt.start_with? "has_model_ssim:" } if search_params[:fq]
search_params[:fq] ||= []
search_params[:fq] << "has_model_ssim:Article"
search_params[:sort] = "rand#{(0...8).map { (65 + rand(26)).chr }.join} asc"
results = SolrSearcher.query search_params
results = results["response"]["docs"].map do |solr_doc|
case solr_doc["has_model_ssim"]
when ["Article"]
Article.from_solr_doc solr_doc
when ["Issue"]
Issue.from_solr_doc solr_doc
end
end
render json: { content: render_to_string(layout: false, partial: "random_sample", locals: { resulting_docs: results }) }
end
private
private
def strip_input_fields
params.each do |key, value|
params[key] = value.strip if value.respond_to?("strip")
end
def strip_input_fields
params.each do |key, value|
params[key] = value.strip if value.respond_to?("strip")
end
end
end
require "openssl"
require "base64"
module Authentication
extend ActiveSupport::Concern
included do
before_action :current_user
helper_method :current_user
helper_method :decrypt_header
end
def authenticate_user!
# user_dict = { "labs_user_id" => decrypt_header(request.headers["X-Auth-Newseye-Token"])[0],
# "labs_user_name" => decrypt_header(request.headers["X-Auth-Newseye-Token"])[1] }
user_dict = { "labs_user_id" => "42", "labs_user_name" => "dummyuser" }
@user = User.find_by(labs_user_id: user_dict["labs_user_id"])
if @user
if !session[:current_user_id]
# puts "Logging in the user since current_user_id was not set"
login @user
end
else
@user = User.new(user_dict)
if @user.save
login @user
end
end
end
def login(user)
reset_session
session[:current_user_id] = user.id
# puts "The user id taken from session is " + String(session[:current_user_id])
end
private
def current_user
# Current.user ||= User.find_by(labs_user_id: decrypt_header(request.headers["X-Auth-Newseye-Token"])[0])
Current.user ||= User.find_by(labs_user_id: "42")
end
def decrypt_header(token)
private_key = ENV["NEP_AUTH_PRIVATE_KEY"] || "OFE_GQ8Ri8MX-0rH_T0e9ZFIhy-q0n2VxBWPoOyJ1I0="
unpacked_key = Base64.urlsafe_decode64(private_key)
signing_key = unpacked_key[0..15]
encryption_key = unpacked_key[16..32]
begin
unpacked_token = Base64.urlsafe_decode64(token)
version = unpacked_token[0]
timestamp = unpacked_token[1..8]
iv = unpacked_token[9..24]
ciphertext = unpacked_token[25..-33]
hmac = unpacked_token[-32..-1]
computed_hmac = OpenSSL::HMAC.digest("SHA256", signing_key, unpacked_token[0..-33])
if OpenSSL.fixed_length_secure_compare(version, "\x80") &&
(timestamp.unpack("Q>")[0] - Time.now.to_i).abs < 3600 &&
OpenSSL.fixed_length_secure_compare(hmac, computed_hmac)
# all good
# puts "Token authenticated"
else
# do something now
raise "Invalid Token"
end
d = OpenSSL::Cipher.new("AES-128-CBC")
d.decrypt
d.key = encryption_key
d.iv = iv
plain = d.update(ciphertext) + d.final
token_tuple = plain.split(",", 2)
rescue => e
# do something now
puts e.message, e.backtrace
raise "Invalid Token"
end
end
end
class DatasetController < ApplicationController
before_action :authenticate_user!
##
# List all datasets
def index
end
##
# Display a single dataset
def show
@dataset = Dataset.find(params[:id])
@current_page = params[:page] || 1
@per_page = params[:per_page] || 10
session[:working_dataset] = @dataset.id
end
##
# Create a new empty dataset
def create_dataset
dataset = Dataset.new
dataset.user = current_user
dataset.title = params[:title]
begin
dataset.save!
render json: {status: 'ok'}
rescue ActiveRecord::RecordNotUnique
render json: {status: "error", message: "A dataset with this title already exists."}
rescue ActiveRecord::RecordInvalid
render json: {status: "error", message: "The title should not be blank."}
end
end
##
# Rename an existing dataset
def rename_dataset
dataset = Dataset.find(params[:id])
dataset.title = params[:title]
before_action :authenticate_user!
##
# List all datasets
def index
# puts "Listing datasets"
# puts session.inspect
end
##
# Display a single dataset
def show
# puts "Finding dataset with id " + String(params[:id])
@dataset = Dataset.find(params[:id])
@current_page = params[:page] || 1
@per_page = params[:per_page] || 10
@nb_pages = params[:nb_pages] || (@dataset.nb_articles / @per_page).to_i + 1
session[:working_dataset] = @dataset.id
# puts "The session now has working_dataset " + String(session[:working_dataset])
end
##
# Create a new empty dataset
def create_dataset
dataset = Dataset.new
dataset.user = current_user
dataset.title = params[:title]
begin
dataset.save!
render json: { status: "ok" }
rescue ActiveRecord::RecordNotUnique
render json: { status: "error", message: "A dataset with this title already exists." }
rescue ActiveRecord::RecordInvalid
render json: { status: "error", message: "The title should not be blank." }
end
end
##
# Rename an existing dataset
def rename_dataset
dataset = Dataset.find(params[:id])
dataset.title = params[:title]
begin
dataset.save!
render json: { status: "ok" }
rescue ActiveRecord::RecordNotUnique
render json: { status: "error", message: "A dataset with this title already exists." }
rescue ActiveRecord::RecordInvalid
render json: { status: "error", message: "The title should not be blank." }
end
end
##
# Import a public dataset
def import_dataset
to_copy = Dataset.find params[:original_dataset_id]
render json: { status: "error", message: "This dataset is not public." } unless to_copy.public?
new_dataset = Dataset.new
new_dataset.user_id = current_user.id
new_dataset.title = params[:title]
to_copy.documents.each do |doc|
if doc["type"] == "compound"
ca = CompoundArticle.find(doc["id"]).dup
ca.user = current_user
begin
dataset.save!
render json: {status: 'ok'}
ca.save!
rescue ActiveRecord::RecordNotUnique
render json: {status: "error", message: "A dataset with this title already exists."}
rescue ActiveRecord::RecordInvalid
render json: {status: "error", message: "The title should not be blank."}
ca.title = "_#{(0...8).map { (65 + rand(26)).chr }.join}_#{ca.title}"
ca.save!
end
end
##
# Import a public dataset
def import_dataset
to_copy = Dataset.find params[:original_dataset_id]
render json: {status: "error", message: "This dataset is not public."} unless to_copy.public?
new_dataset = Dataset.new
new_dataset.user_id = current_user.id
new_dataset.title = params[:title]
to_copy.documents.each do |doc|
if doc['type'] == "compound"
ca = CompoundArticle.find(doc['id']).dup
ca.user = current_user
begin
ca.save!
rescue ActiveRecord::RecordNotUnique
ca.title = "_#{(0...8).map { (65 + rand(26)).chr }.join}_#{ca.title}"
ca.save!
end
new_dataset.documents << {id: ca.id, type: "compound"}
else
new_dataset.documents << doc
end
end
begin
new_dataset.save!
render json: {status: 'ok'}
rescue ActiveRecord::RecordNotUnique
render json: {status: "error", message: "A dataset with this title already exists."}
rescue ActiveRecord::RecordInvalid
render json: {status: "error", message: "The title should not be blank."}
end
end
##
# Delete an existing dataset
def delete_dataset
dataset = Dataset.find(params[:dataset_id])
dataset_id = dataset.id
dataset.destroy
if session[:working_dataset] == dataset_id
if current_user.datasets.first
session[:working_dataset] = current_user.datasets.first.id
else
session[:working_dataset] = nil
end
end
end
##
# Update the view of the list of datasets
def update_datasets_list
respond_to do |format|
format.js
end
end
##
#
def set_working_dataset
session[:working_dataset] = params[:dataset_id]
@title = Dataset.find(session[:working_dataset]).title
respond_to do |format|
format.js
end
end
def add_selected_documents
out = {}
@nb_added_docs = params[:documents_ids].size
dataset = Dataset.find(session[:working_dataset])
existing = dataset.add_documents params[:documents_ids] # Add docs and return existing ids
@nb_added_docs -= existing.size
title = dataset.title
message = "<p> #{@nb_added_docs} document#{@nb_added_docs > 1 ? "s were" : " was"} added to your dataset.</p>"
message.concat "<p>#{existing.size} document#{existing.size > 1 ? "s" : ""} already exist in this dataset.</p>" unless existing.empty?
# render partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
out['notif'] = render_to_string layout: false, partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
out['nbissues'] = dataset.documents.select{|d| d['type'] == "issue" }.size
out['nbarticles'] = dataset.documents.select{|d| d['type'] == "article" }.size
out['nbdocs'] = out['nbissues'] + out['nbarticles']
out['title'] = title
out['results_datasets'] = params[:documents_ids].map{ |docid| [docid, render_to_string(layout: false, partial: 'catalog/result_datasets', locals: {doc_id: docid})] }.to_h
render json: out
end
def add_compound
out = {}
dataset = Dataset.find(session[:working_dataset])
existing = dataset.add_compound params[:compound_id] # Add docs and return existing ids
title = dataset.title
message = "<p> The compound article was added to your dataset.</p>"
out['notif'] = render_to_string layout: false, partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
out['nbissues'] = dataset.documents.select{|d| d['type'] == "issue" }.size
out['nbarticles'] = dataset.documents.select{|d| d['type'] == "article" }.size
out['nbcompounds'] = dataset.documents.select{|d| d['type'] == "compound" }.size
out['nbdocs'] = out['nbissues'] + out['nbarticles'] + out['nbcompounds']
out['title'] = title
render json: out
end
def remove_selected_documents
@nb_removed_docs = params[:documents_ids].size
dataset = Dataset.find(session[:working_dataset])
dataset.remove_documents params[:documents_ids]
redirect_to action: "show", id: dataset.id
end
def add_all_documents
SearchToDatasetWorker.perform_async(current_user.id, session[:working_dataset], params[:search_params].to_unsafe_h)
title = Dataset.find(session[:working_dataset]).title
message = "<p>Documents are being added to your dataset. You will be notified when the operation is done.</p>"
render partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
end
def export_dataset
ExportDatasetWorker.perform_async(current_user.id, params[:dataset_id], params[:export_type])
title = Dataset.find(params[:dataset_id]).title
message = "<p>The export is being prepared. You will be notified when the operation is done.</p>"
render partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
end
def toggle_sharing_status
@dataset = Dataset.find(params[:dataset_id])
@dataset.toggle!(:public)
render partial: 'dataset_info'
end
def paginate
out = {}
d = Dataset.find params['id']
rows = params[:per_page].to_i
res = d.fetch_paginated_documents(params[:page].to_i, rows, params[:sort], params[:sort_order], params[:type])
out[:documents] = render_to_string(layout: false,
partial: "documents",
locals: {docs: res[:docs], rows: rows, pagenum: params[:page].to_i})
out[:pagination] = render_to_string(layout: false,
partial: "pagination",
locals: {nb_pages: res[:nb_pages].to_i, current_page: params[:page].to_i})
render json: out
end
def list_datasets
render json: current_user.datasets.to_json
end
new_dataset.documents << { id: ca.id, type: "compound" }
else
new_dataset.documents << doc
end
end
begin
new_dataset.save!
render json: { status: "ok" }
rescue ActiveRecord::RecordNotUnique
render json: { status: "error", message: "A dataset with this title already exists." }
rescue ActiveRecord::RecordInvalid
render json: { status: "error", message: "The title should not be blank." }
end
end
##
# Delete an existing dataset
def delete_dataset
dataset = Dataset.find(params[:dataset_id])
dataset_id = dataset.id
dataset.destroy
if session[:working_dataset] == dataset_id
if current_user.datasets.first
session[:working_dataset] = current_user.datasets.first.id
else
session[:working_dataset] = nil
end
end
end
##
# Update the view of the list of datasets
def update_datasets_list
respond_to do |format|
format.js
end
end
##
#
def set_working_dataset
session[:working_dataset] = params[:dataset_id]
@title = Dataset.find(session[:working_dataset]).title
# puts "Setting working dataset to " + String(session[:working_dataset])
# puts session.inspect
respond_to do |format|
format.js
end
end
def add_selected_documents
# puts session.inspect
# puts "User id from session is " + String(session[:current_user_id]) + " and working dataset is " + String(session[:working_dataset])
out = {}
@nb_added_docs = params[:documents_ids].size
dataset = Dataset.find(session[:working_dataset])
existing = dataset.add_documents params[:documents_ids] # Add docs and return existing ids
@nb_added_docs -= existing.size
title = dataset.title
message = "<p> #{@nb_added_docs} document#{@nb_added_docs > 1 ? "s were" : " was"} added to your dataset.</p>"
message.concat "<p>#{existing.size} document#{existing.size > 1 ? "s" : ""} already exist in this dataset.</p>" unless existing.empty?
# render partial: "shared/notification", locals: {notif_title: title, notif_content: message.html_safe}
out["notif"] = render_to_string layout: false, partial: "shared/notification", locals: { notif_title: title, notif_content: message.html_safe, notif_autohide: "true" }
out["nbissues"] = dataset.documents.select { |d| d["type"] == "issue" }.size
out["nbarticles"] = dataset.documents.select { |d| d["type"] == "article" }.size
out["nbdocs"] = out["nbissues"] + out["nbarticles"]
out["title"] = title
out["results_datasets"] = params[:documents_ids].map { |docid| [docid, render_to_string(layout: false, partial: "catalog/result_datasets", locals: { doc_id: docid })] }.to_h
render json: out
end
def add_compound
out = {}
dataset = Dataset.find(session[:working_dataset])
existing = dataset.add_compound params[:compound_id] # Add docs and return existing ids
title = dataset.title
message = "<p> The compound article was added to your dataset.</p>"
out["notif"] = render_to_string layout: false, partial: "shared/notification", locals: { notif_title: title, notif_content: message.html_safe, notif_autohide: "true" }
out["nbissues"] = dataset.documents.select { |d| d["type"] == "issue" }.size
out["nbarticles"] = dataset.documents.select { |d| d["type"] == "article" }.size
out["nbcompounds"] = dataset.documents.select { |d| d["type"] == "compound" }.size
out["nbdocs"] = out["nbissues"] + out["nbarticles"] + out["nbcompounds"]
out["title"] = title
render json: out
end
def remove_selected_documents
@nb_removed_docs = params[:documents_ids].size
dataset = Dataset.find(session[:working_dataset])
dataset.remove_documents params[:documents_ids]
respond_to do |format|
format.js {render inline: "location.reload();" } # https://stackoverflow.com/questions/7465259/how-can-i-reload-the-current-page-in-ruby-on-rails
end
end
def add_all_documents
time = Time.now.to_i
SearchToDatasetWorker.perform_async(current_user.id, session[:working_dataset], params[:search_params].to_unsafe_h, time)
title = Dataset.find(session[:working_dataset]).title
message = "<p>Documents are being added to your dataset.</p><div class=\"completion-rate mt-2\">
<div class=\"progress\" id=\"progress-#{session[:working_dataset]}#{time}\">
<div class=\"progress-bar progress-bar-striped\" role=\"progressbar\" style=\"width: 0%;\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\">0%</div>
</div>
</div>"
render partial: "shared/notification", locals: { notif_title: title, notif_content: message.html_safe, notif_autohide: "false" }
end
def export_dataset
time = Time.now.to_i
ExportDatasetWorker.perform_async(current_user.id, params[:dataset_id], params[:export_type], time)
title = Dataset.find(params[:dataset_id]).title
message = "<p>The export is being prepared.</p><div class=\"completion-rate mt-2\">
<div class=\"progress\" id=\"progress-#{session[:working_dataset]}#{time}\">
<div class=\"progress-bar progress-bar-striped\" role=\"progressbar\" style=\"width: 0%;\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\">0%</div>
</div>
</div>"
render partial: "shared/notification", locals: { notif_title: title, notif_content: message.html_safe, notif_autohide: "false" }
end
def toggle_sharing_status
@dataset = Dataset.find(params[:dataset_id])
@dataset.toggle!(:public)
render partial: "dataset_info"
end
def paginate
out = {}
d = Dataset.find params["id"]
rows = params[:per_page].to_i
res = d.fetch_paginated_documents(params[:page].to_i, rows, params[:sort], params[:sort_order], params[:type])
out[:documents] = render_to_string(layout: false,
partial: "documents",
locals: { docs: res[:docs], rows: rows, pagenum: params[:page].to_i })
out[:pagination] = render_to_string(layout: false,
partial: "pagination",
locals: { nb_pages: res[:nb_pages].to_i, current_page: params[:page].to_i })
out[:nb_pages] = res[:nb_pages]
render json: out
end
def list_datasets
render json: current_user.datasets.to_json
end
end
class ExperimentController < ApplicationController
before_action :authenticate_user!
before_action :authenticate_user!
def index
end
def index
def create
experiment = Experiment.new
experiment.user = current_user
experiment.title = params[:title]
begin
experiment.save!
render json: { status: "ok" }
rescue ActiveRecord::RecordNotUnique
render json: { status: "error", message: "An experiment with this title already exists." }
rescue ActiveRecord::RecordInvalid
render json: { status: "error", message: "The title should not be blank." }
end
end
def create
experiment = Experiment.new
experiment.user = current_user
experiment.title = params[:title]
begin
experiment.save!
render json: {status: 'ok'}
rescue ActiveRecord::RecordNotUnique
render json: {status: "error", message: "An experiment with this title already exists."}
rescue ActiveRecord::RecordInvalid
render json: {status: "error", message: "The title should not be blank."}
end
def delete
experiment = Experiment.find(params[:experiment_id])
root_ids = experiment.description["children"].map { |root| root["tool"]["id"] }
root_ids.each do |root_id|
Tool.destroy(experiment.delete_tool(root_id))
end
experiment.destroy
end
def delete
experiment = Experiment.find(params[:experiment_id])
root_ids = experiment.description["children"].map{|root| root['tool']['id'] }
root_ids.each do |root_id|
Tool.destroy(experiment.delete_tool(root_id))
end
experiment.destroy
end
def show
@experiment = Experiment.find params[:id]
@tools = @experiment.load_tools
@tools = JSON.parse(File.read("#{Rails.root}/lib/newspapers_tools.json"))
@tools['tools']['processors'].delete_if{ |h| h["type"] == "splitter" }
end
def show
@experiment = Experiment.find params[:id]
@tools = @experiment.load_tools
@tools = JSON.parse(File.read("#{Rails.root}/lib/newspapers_tools.json"))
@tools["tools"]["processors"].delete_if { |h| h["type"] == "splitter" }
end
def update_experiments_list
respond_to do |format|
format.js
end
def update_experiments_list
respond_to do |format|
format.js
end
end
def add_tool
@experiment = Experiment.find(params[:id])
tool_params = JSON.parse params[:tool]
tool = Tool.new
tool.tool_type = tool_params['type']
tool.input_type = tool_params['input_type']
tool.output_type = tool_params['output_type']
tool.parameters = tool_params['parameters']
tool.status = "created"
tool.parent_id = params[:parent_id]#(params[:parent_id] == "") ? nil : Tool.find(params[:parent_id])
tool.experiment = @experiment
tool.save!
@experiment.add_tool(params[:parent_id].to_i, tool)
@experiment.save!
render 'experiment/update_experiment_area'
end
def add_tool
@experiment = Experiment.find(params[:id])
tool_params = JSON.parse params[:tool]
tool = Tool.new
tool.tool_type = tool_params["type"]
tool.input_type = tool_params["input_type"]
tool.output_type = tool_params["output_type"]
tool.parameters = tool_params["parameters"]
tool.status = "created"
tool.parent_id = params[:parent_id] #(params[:parent_id] == "") ? nil : Tool.find(params[:parent_id])
tool.experiment = @experiment
tool.save!
@experiment.add_tool(params[:parent_id].to_i, tool)
@experiment.save!
render "experiment/update_experiment_area"
end
def delete_tool
@experiment = Experiment.find(params[:id])
tools_to_destroy_ids = @experiment.delete_tool(params[:tool_id].to_i)
@experiment.save!
Tool.destroy(tools_to_destroy_ids)
render 'experiment/update_experiment_area'
end
def delete_tool
@experiment = Experiment.find(params[:id])
tools_to_destroy_ids = @experiment.delete_tool(params[:tool_id].to_i)
@experiment.save!
Tool.destroy(tools_to_destroy_ids)
render "experiment/update_experiment_area"
end
def edit_tool_form
@tool = Tool.find(params[:tool_id])
render partial: 'tool/parameters', locals: {tool: @tool}
end
def edit_tool_form
@tool = Tool.find(params[:tool_id])
render partial: "tool/parameters", locals: { tool: @tool }
end
def edit_tool
@experiment = Experiment.find(params[:id])
@tool = Tool.find(params[:tool_id])
modified = false
@tool.parameters.map! do |param|
if param['value'] != params[:parameters][param['name']]
modified = true
end
param['value'] = params[:parameters][param['name']]
param
end
@tool.status = "configured" if modified
@tool.save!
render 'experiment/update_experiment_area'
def edit_tool
@experiment = Experiment.find(params[:id])
@tool = Tool.find(params[:tool_id])
modified = false
@tool.parameters.map! do |param|
if param["value"] != params[:parameters][param["name"]]
modified = true
end
param["value"] = params[:parameters][param["name"]]
param
end
@tool.status = "configured" if modified
@tool.save!
render "experiment/update_experiment_area"
end
def tool_results
@experiment = Experiment.find(params[:id])
@tool = Tool.find(params[:tool_id])
render partial: 'tool/results', locals: {tool: @tool, experiment: @experiment}
end
def tool_results
@experiment = Experiment.find(params[:id])
@tool = Tool.find(params[:tool_id])
render partial: "tool/results", locals: { tool: @tool, experiment: @experiment }
end
def run_tool
@experiment = Experiment.find(params[:id])
@tool = Tool.find(params[:tool_id])
@tool.run()
render 'experiment/update_experiment_area'
end
def run_tool
@experiment = Experiment.find(params[:id])
@tool = Tool.find(params[:tool_id])
@tool.run()
render "experiment/update_experiment_area"
end
def run_experiment
out = {}
@experiment = Experiment.find(params[:experiment_id])
ids = @experiment.get_tool_ids
running = false
ids.map{|id| Tool.find(id)}.each do |tool|
if tool.runnable?
tool.run(true)
running = true
end
end
out[:html_tree] = render_to_string partial: "tree", locals: {experiment: @experiment}
out[:experiment_running] = running
render json: out
def run_experiment
out = {}
@experiment = Experiment.find(params[:experiment_id])
ids = @experiment.get_tool_ids
running = false
ids.map { |id| Tool.find(id) }.each do |tool|
if tool.runnable?
tool.run(true)
running = true
end
end
out[:html_tree] = render_to_string partial: "tree", locals: { experiment: @experiment }
out[:experiment_running] = running
render json: out
end
end
class NotificationController < ApplicationController
end
\ No newline at end of file
before_action :authenticate_user!
end
class ToolController < ApplicationController
before_action :authenticate_user!
before_action :authenticate_user!
def show
end
def show
def create
end
end
def update
end
def create
def destroy
end
end
private
def update
end
def destroy
end
private
def tool_params
params.require(:tool).permit(:parameters, :results, :status)
end
def tool_params
params.require(:tool).permit(:parameters, :results, :status)
end
end
module ApplicationHelper
def set_page_title(title)
content_for :page_title, title
end
def set_page_title(title)
content_for :page_title, title
end
end
module ExperimentHelper
def recursive_display(tree, tools)
if tree.has_key? "tool"
concat "<li>".html_safe
concat render partial: 'tool/canvas_tool', locals: {tool: tools[tree['tool']['id']]}
concat "<ul>".html_safe
end
tree['children'].each do |node|
recursive_display(node, tools)
end
concat '<li><div class="tf-nc tool-slot dnd-zone"></div></li>'.html_safe
concat "</ul>".html_safe
concat "</li>".html_safe if tree.has_key? "tool"
def recursive_display(tree, tools)
if tree.has_key? "tool"
concat "<li>".html_safe
concat render partial: "tool/canvas_tool", locals: { tool: tools[tree["tool"]["id"]] }
concat "<ul>".html_safe
end
end
\ No newline at end of file
tree["children"].each do |node|
recursive_display(node, tools)
end
concat '<li><div class="tf-nc tool-slot dnd-zone"></div></li>'.html_safe
concat "</ul>".html_safe
concat "</li>".html_safe if tree.has_key? "tool"
end
end
module NamedEntitiesHelper
def get_linked_entities entities
priority_language = [I18n.locale, 'en', 'de', 'fr', 'fi', 'sv']
ids = entities.select{ |label| label != "" && label != nil }
return {} if ids.empty?
out = {}
SolrSearcher.query({q: "*:*", fq: "id:(#{ids.join(' ')})", fl: "*", rows: 99999})['response']['docs'].map do |res|
priority_language.each do |lang|
unless res["label_#{lang}_ssi"].nil?
out[res['id']] = {kb_url: res['kb_url_ssi'], label: res["label_#{lang}_ssi"]}
break
end
end
def get_linked_entities(entities)
priority_language = [I18n.locale, "en", "de", "fr", "fi", "sv"]
ids = entities.select { |label| label != "" && label != nil }
return {} if ids.empty?
out = {}
SolrSearcher.query({ q: "*:*", fq: "id:(#{ids.join(" ")})", fl: "*", rows: 99999 })["response"]["docs"].map do |res|
priority_language.each do |lang|
unless res["label_#{lang}_ssi"].nil?
out[res["id"]] = { kb_url: res["kb_url_ssi"], label: res["label_#{lang}_ssi"] }
break
end
out
end
end
out
end
def get_entity_label(options={})
priority_language = [I18n.locale, 'en', 'de', 'fr', 'fi', 'sv']
if options.class == Array
out = {}
unless options.empty?
docs = SolrSearcher.query({q: "*:*", fq: "id:(#{options.join(' ')})", fl: "*", rows: 99999})['response']['docs']
docs.map do |doc|
priority_language.each do |lang|
unless doc["label_#{lang}_ssi"].nil?
out[doc['id']] = doc["label_#{lang}_ssi"]
break
end
end
end
def get_entity_label(options = {})
priority_language = [I18n.locale, "en", "de", "fr", "fi", "sv"]
if options.class == Array
out = {}
unless options.empty?
docs = SolrSearcher.query({ q: "*:*", fq: "id:(#{options.join(" ")})", fl: "*", rows: 99999 })["response"]["docs"]
docs.map do |doc|
priority_language.each do |lang|
unless doc["label_#{lang}_ssi"].nil?
out[doc["id"]] = doc["label_#{lang}_ssi"]
break
end
return out
else
@entities_labels[options] # set in catalog_controller#index
end
end
end
return out
else
@entities_labels[options] # set in catalog_controller#index
end
end
\ No newline at end of file
end
end
module SearchHelper
def current_page_params
params.to_unsafe_h.slice("q", "page", "per_page", "sort", "search_type", "f")
end
def current_page_params
params.to_unsafe_h.slice('q', 'page', 'per_page','sort', 'search_type', 'f')
def merge_facets(parameters, new)
parameters.merge(new) do |key, oldval, newval|
oldval.merge(newval)
end
end
def merge_facets(parameters, new)
parameters.merge(new) do |key, oldval, newval|
oldval.merge(newval)
end
end
def convert_solr_date_to_datepicker_date solr_date
DateTime.parse(solr_date).strftime("%Y-%m-%d")
end
def convert_solr_date_to_datepicker_date(solr_date)
DateTime.parse(solr_date).strftime("%Y-%m-%d")
end
def convert_datepicker_date_to_solr_date solr_date
DateTime.parse(solr_date).strftime("%Y-%m-%d")
end
def convert_datepicker_date_to_solr_date(solr_date)
DateTime.parse(solr_date).strftime("%Y-%m-%d")
end
def search_constraints
constraints = []
if current_page_params[:f]
current_page_params[:f].each do |f, vals|
if f == "date_created_dtsi"
constraints << {label: f, value: "From #{vals['from']} To #{vals['to']}"}
else
vals.each do |val|
constraints << {label: f, value: val}
end
end
end
def search_constraints
constraints = []
if current_page_params[:f]
current_page_params[:f].each do |f, vals|
if f == "date_created_dtsi"
constraints << { label: f, value: "From #{vals["from"]} To #{vals["to"]}" }
else
vals.each do |val|
constraints << { label: f, value: val }
end
end
constraints
end
end
end
\ No newline at end of file
constraints
end
end
......@@ -19,7 +19,7 @@ consumer.subscriptions.create("NotificationChannel", {
$("#experiment_area").attr("data-refresh", (!$("#experiment_area").attr("data-refresh")))
break
case "notify":
if(window.location.pathname == "/search") {
if(window.location.pathname.endsWith("/search")) {
const selected_dataset = $("#working_dataset_select").val()
$("#working_dataset_select").html(data.dataset_options)
$("#working_dataset_select").val(selected_dataset)
......@@ -35,12 +35,22 @@ consumer.subscriptions.create("NotificationChannel", {
}
break
case "completion_rate":
if(window.location.pathname == `/experiment/${data.experiment_id}`) {
if(window.location.pathname.endsWith(`/experiment/${data.experiment_id}`)) {
const progress_bar = $(`#tool_${data.tool_id}`).find(".completion-rate").find('.progress-bar')
progress_bar.attr("style", `width: ${data.completion}%;`)
progress_bar.attr("aria-valuenow", data.completion)
progress_bar.html(`${data.completion}%`)
}
if(window.location.pathname.endsWith("/search") || window.location.pathname.endsWith("/dataset/".concat(data.dataset_id))) {
const progress_bar = $("#progress-".concat(data.dataset_id).concat(data.time)).find('.progress-bar')
progress_bar.attr("style", `width: ${data.completion}%;`)
progress_bar.attr("aria-valuenow", data.completion)
progress_bar.html(`${data.completion}%`)
if(data.completion == 100) {
progress_bar.closest(".toast").hide(2000);
}
}
break
case "experiment_finished":
// $("#experiment_status").html(data.message)
......
......@@ -27,5 +27,15 @@ window.$ = $
window.bootstrap = bootstrap
window.Panzoom = require('@panzoom/panzoom')
import "./application.scss"
import "./stylesheets/catalog.scss";
const images = require.context('../images', true)
import Chart from 'chart.js/auto'
export function addPrefixURL() {
var prefix = document.querySelector('#newspaper-platform');
if (prefix != null) {
return prefix.dataset.prefix;
} else {
return "";
}
}
$labs-green: hsl(180, 15%, 30%);
$dark-labs-green: hsl(180, 5%, 35%);
$theme-colors: ( // Overwrite bootstrap colors
'primary': $labs-green,
'secondary': #747474,
'success': $dark-labs-green,
'danger': lighten($labs-green, 20%),
'warning': lighten($labs-green, 25%),
'info': lighten($labs-green, 30%),
'light': white,
'dark': #525e69);
$link-color: $labs-green;
$link-hover-color: darken($labs-green, 10%) !default;
$component-active-bg: $labs-green;
$card-color: black;
$border-color: #69737e;
$fa-font-path: '../../../node_modules/@fortawesome/fontawesome-free/webfonts';
@import '~@fortawesome/fontawesome-free/scss/fontawesome';
......@@ -9,5 +26,3 @@ $fa-font-path: '../../../node_modules/@fortawesome/fontawesome-free/webfonts';
@import '~bootstrap/scss/bootstrap';
@import "~treeflex/dist/css/treeflex";
@import "./stylesheets/catalog.scss";
\ No newline at end of file
......@@ -3,8 +3,8 @@ import {DatasetAPI} from "../utils/dataset_api"
import {SearchAPI} from "../utils/search_api";
export default class extends Controller {
static targets = [ ]
static values = { id: Number, currentPage: Number, perPage: Number, sort: String, sortOrder: String, selectedDocuments: Array }
static targets = [ "inputPage" ]
static values = { id: Number, currentPage: Number, nbPages: Number, perPage: Number, sort: String, sortOrder: String, selectedDocuments: Array }
connect() {
this.loadDocuments(this.idValue, this.currentPageValue, this.perPageValue, this.sortValue, this.sortOrderValue, "article")
......@@ -28,9 +28,17 @@ export default class extends Controller {
}
toggleSharingStatus(event) {
DatasetAPI.toggleSharingStatus(this.idValue, (data) => {
document.getElementById("dataset-info").outerHTML= data
})
if (document.getElementById("sharing_status").innerText == "Private") {
if (confirm("By clicking OK you agree that your username will be publicly displayed next to your dataset")) {
DatasetAPI.toggleSharingStatus(this.idValue, (data) => {
document.getElementById("dataset-info").outerHTML = data
})
}
} else {
DatasetAPI.toggleSharingStatus(this.idValue, (data) => {
document.getElementById("dataset-info").outerHTML = data
})
}
}
export(event) {
......@@ -59,6 +67,7 @@ export default class extends Controller {
DatasetAPI.paginateDataset(datasetId, page, per_page, sort, sort_order, type, (data) => {
$("#documents-list").html(data.documents)
$("#results_navigation").html(data.pagination)
this.nbPagesValue = data.nb_pages
})
}
......@@ -72,7 +81,7 @@ export default class extends Controller {
event.preventDefault()
if (this.currentPageValue > 1) {
this.currentPageValue--
this.loadDocuments(this.idValue, this.currentPageValue, this.perPageValue, this.sortValue, this.sortOrderValue, "all")
this.loadDocuments(this.idValue, this.currentPageValue, this.perPageValue, this.sortValue, this.sortOrderValue, $("#doctype_selection input:checked").data("doctype"))
}
}
......@@ -80,14 +89,22 @@ export default class extends Controller {
event.preventDefault()
if (this.currentPageValue < this.nbPagesValue) {
this.currentPageValue++
this.loadDocuments(this.idValue, this.currentPageValue, this.perPageValue, this.sortValue, this.sortOrderValue, "all")
this.loadDocuments(this.idValue, this.currentPageValue, this.perPageValue, this.sortValue, this.sortOrderValue, $("#doctype_selection input:checked").data("doctype"))
}
}
page_button(event) {
event.preventDefault()
this.currentPageValue = event.target.textContent
this.loadDocuments(this.idValue, this.currentPageValue, this.perPageValue, this.sortValue, this.sortOrderValue, "all")
this.loadDocuments(this.idValue, this.currentPageValue, this.perPageValue, this.sortValue, this.sortOrderValue, $("#doctype_selection input:checked").data("doctype"))
}
page_select(event) {
event.preventDefault()
const input_page = parseInt(this.inputPageTarget.value)
if (!isNaN(input_page) && 1 <= input_page && input_page <= this.nbPagesValue && !(this.currentPageValue == input_page)) {
this.currentPageValue = input_page
this.loadDocuments(this.idValue, this.currentPageValue, this.perPageValue, this.sortValue, this.sortOrderValue, $("#doctype_selection input:checked").data("doctype"))
}
}
}
\ No newline at end of file