From 0d1f9fea511b669450359fd86f2b27f6632d66bb Mon Sep 17 00:00:00 2001 From: axel Date: Thu, 16 Sep 2021 23:21:00 +0200 Subject: [PATCH] Setup ActionCable/ActiveJob using Sidekiq/Redis + Experiment/Tool refont --- Gemfile | 1 + Gemfile.lock | 7 + README.md | 15 +- app/channels/application_cable/connection.rb | 15 ++ app/channels/notification_channel.rb | 9 ++ app/controllers/experiment_controller.rb | 65 +++++++- app/controllers/tool_controller.rb | 26 ++++ app/helpers/experiment_helper.rb | 17 +++ .../channels/notification_channel.js | 17 +++ .../packs/controllers/datasets_controller.js | 2 + .../controllers/experiment_controller.js | 140 +++++------------- .../controllers/experiments_controller.js | 46 ++++++ .../packs/controllers/hello_controller.js | 10 -- app/javascript/packs/utils/server_api.js | 126 ++++++++++++---- app/javascript/packs/utils/templates.js | 9 +- app/models/experiment.rb | 80 +++++++++- app/models/tool.rb | 16 ++ app/views/experiment/_canvas_tool.html.erb | 17 --- .../experiment/_experiment_menu.html.erb | 8 + .../experiment/_experiments_list.html.erb | 16 ++ app/views/experiment/_tools_menu.html.erb | 29 ++++ app/views/experiment/_tree.html.erb | 5 + app/views/experiment/index.html.erb | 45 +++--- app/views/experiment/show.html.erb | 51 +------ .../experiment/update_experiment_area.js.erb | 1 + .../experiment/update_experiments_list.js.erb | 1 + app/views/layouts/application.html.erb | 1 + .../named_entities/_named_entities.html.erb | 0 app/views/tool/_canvas_tool.html.erb | 19 +++ .../{experiment => tool}/_menu_tool.html.erb | 0 app/views/tool/_parameters.html.erb | 37 +++++ app/workers/tool_runner_worker.rb | 7 + config/application.rb | 26 ++-- config/cable.yml | 6 +- config/initializers/warden_hooks.rb | 9 ++ config/locales/newspapers.en.yml | 5 +- config/routes.rb | 51 ++++--- .../20210903194218_create_experiment.rb | 3 +- db/migrate/20210915140752_add_tool.rb | 12 ++ db/schema.rb | 17 ++- test/channels/notification_channel_test.rb | 8 + test/workers/tool_runner_worker_test.rb | 6 + 42 files changed, 711 insertions(+), 270 deletions(-) create mode 100644 app/channels/notification_channel.rb create mode 100644 app/controllers/tool_controller.rb create mode 100644 app/helpers/experiment_helper.rb create mode 100644 app/javascript/channels/notification_channel.js create mode 100644 app/javascript/packs/controllers/experiments_controller.js delete mode 100644 app/javascript/packs/controllers/hello_controller.js create mode 100644 app/models/tool.rb delete mode 100644 app/views/experiment/_canvas_tool.html.erb create mode 100644 app/views/experiment/_experiment_menu.html.erb create mode 100644 app/views/experiment/_experiments_list.html.erb create mode 100644 app/views/experiment/_tools_menu.html.erb create mode 100644 app/views/experiment/_tree.html.erb create mode 100644 app/views/experiment/update_experiment_area.js.erb create mode 100644 app/views/experiment/update_experiments_list.js.erb create mode 100644 app/views/named_entities/_named_entities.html.erb create mode 100644 app/views/tool/_canvas_tool.html.erb rename app/views/{experiment => tool}/_menu_tool.html.erb (100%) create mode 100644 app/views/tool/_parameters.html.erb create mode 100644 app/workers/tool_runner_worker.rb create mode 100644 config/initializers/warden_hooks.rb create mode 100644 db/migrate/20210915140752_add_tool.rb create mode 100644 test/channels/notification_channel_test.rb create mode 100644 test/workers/tool_runner_worker_test.rb diff --git a/Gemfile b/Gemfile index 1758131..2f9c1e5 100644 --- a/Gemfile +++ b/Gemfile @@ -25,6 +25,7 @@ gem 'jbuilder', '~> 2.7' # gem 'bcrypt', '~> 3.1.7' gem 'rsolr' +gem 'sidekiq', '~> 6.0' # Use Active Storage variant # gem 'image_processing', '~> 1.2' diff --git a/Gemfile.lock b/Gemfile.lock index 2073640..104aa98 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -78,6 +78,7 @@ GEM xpath (~> 3.2) childprocess (3.0.0) concurrent-ruby (1.1.9) + connection_pool (2.2.5) crass (1.0.6) devise (4.8.0) bcrypt (~> 3.0) @@ -166,6 +167,7 @@ GEM rb-fsevent (0.11.0) rb-inotify (0.10.1) ffi (~> 1.0) + redis (4.4.0) regexp_parser (2.1.1) responders (3.0.1) actionpack (>= 5.0) @@ -189,6 +191,10 @@ GEM childprocess (>= 0.5, < 4.0) rubyzip (>= 1.2.2) semantic_range (3.0.0) + sidekiq (6.2.2) + connection_pool (>= 2.2.2) + rack (~> 2.0) + redis (>= 4.2.0) spring (2.1.1) sprockets (4.0.2) concurrent-ruby (~> 1.0) @@ -244,6 +250,7 @@ DEPENDENCIES rsolr sass-rails (>= 6) selenium-webdriver + sidekiq (~> 6.0) spring turbolinks (~> 5) tzinfo-data diff --git a/README.md b/README.md index 27891ff..65efc25 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,18 @@ Create a docker container and create an empty database named "newspapers" `docker exec -it newspapers_platform_database createdb -U postgres newspapers` -Modify the content of `config/database.yml` accordingly. +Modify the content of `config/database.yml` according to your configuration. + +## Setting up Redis +Used by Sidekiq and Rails + +`docker run --name newspapers_redis -p 127.0.0.1:6379:6379 -d redis` ## Setting up a IIIF server -Cantaloupe \ No newline at end of file +Cantaloupe + + +## Starting the server +`bundle exec sidekiq` + +`rails s` diff --git a/app/channels/application_cable/connection.rb b/app/channels/application_cable/connection.rb index 0ff5442..6b5519a 100644 --- a/app/channels/application_cable/connection.rb +++ b/app/channels/application_cable/connection.rb @@ -1,4 +1,19 @@ module ApplicationCable class Connection < ActionCable::Connection::Base + identified_by :current_user + + def connect + self.current_user = find_user + end + + def find_user + user_id = cookies.signed["user.id"] + current_user = User.find_by(id: user_id) + if current_user + current_user + else + reject_unauthorized_connection + end + end end end diff --git a/app/channels/notification_channel.rb b/app/channels/notification_channel.rb new file mode 100644 index 0000000..206652a --- /dev/null +++ b/app/channels/notification_channel.rb @@ -0,0 +1,9 @@ +class NotificationChannel < ApplicationCable::Channel + def subscribed + stream_from "notifications.#{current_user.id}" + end + + def unsubscribed + # Any cleanup needed when channel is unsubscribed + end +end diff --git a/app/controllers/experiment_controller.rb b/app/controllers/experiment_controller.rb index ed3635d..8055304 100644 --- a/app/controllers/experiment_controller.rb +++ b/app/controllers/experiment_controller.rb @@ -5,19 +5,70 @@ class ExperimentController < ApplicationController def index end + def create + experiment = Experiment.new + experiment.user = current_user + experiment.title = params[:title] + begin + experiment.save! + render json: {status: 'ok'} + rescue ActiveRecord::RecordNotUnique + render json: {status: "error", message: "An experiment with this title already exists."} + rescue ActiveRecord::RecordInvalid + render json: {status: "error", message: "The title should not be blank."} + end + end + def show @experiment = Experiment.find params[:id] + # TODO: I'm not satisfied with the following @tools_description = File.read("#{Rails.root}/public/newspapers_tools.json") end - def save - experiment = Experiment.find(params[:id]) - experiment.description = JSON.parse(params[:description]) - experiment.save + def update_experiments_list + respond_to do |format| + format.js + end + end + + def add_tool + @experiment = Experiment.find(params[:id]) + tool_params = JSON.parse params[:tool] + tool = Tool.new + tool.tool_type = tool_params['type'] + tool.parameters = tool_params['parameters'] + tool.status = "created" + tool.experiment = @experiment + tool.save! + @experiment.add_tool(params[:parent_id].to_i, tool) + @experiment.save! + render 'experiment/update_experiment_area' + end + + def delete_tool + @experiment = Experiment.find(params[:id]) + tools_to_destroy_ids = @experiment.delete_tool(params[:tool_id].to_i) + @experiment.save! + Tool.destroy(tools_to_destroy_ids) + render 'experiment/update_experiment_area' + end + + def edit_tool_form + @experiment = Experiment.find(params[:id]) + @tool = Tool.find(params[:tool_id]) + render partial: 'tool/parameters', locals: {tool: @tool} end - def load - experiment = Experiment.find(params[:id]) - render json: experiment.description.to_json + def edit_tool + @experiment = Experiment.find(params[:id]) + @tool = Tool.find(params[:tool_id]) + @tool.parameters.map! do |param| + param['value'] = params[:parameters][param['name']] + param + end + @tool.save! + @experiment.update_tool(params[:tool_id].to_i, params[:parameters]) + @experiment.save! + render json: {} end end diff --git a/app/controllers/tool_controller.rb b/app/controllers/tool_controller.rb new file mode 100644 index 0000000..d9fbf32 --- /dev/null +++ b/app/controllers/tool_controller.rb @@ -0,0 +1,26 @@ +class ToolController < ApplicationController + + before_action :authenticate_user! + + def show + + end + + def create + + end + + def update + + end + + def destroy + + end + + private + + def tool_params + params.require(:tool).permit(:parameters, :results, :status) + end +end diff --git a/app/helpers/experiment_helper.rb b/app/helpers/experiment_helper.rb new file mode 100644 index 0000000..352a16d --- /dev/null +++ b/app/helpers/experiment_helper.rb @@ -0,0 +1,17 @@ +module ExperimentHelper + + def recursive_display(tree) + if tree.has_key? "tool" + concat "
  • ".html_safe + concat render partial: 'tool/canvas_tool', locals: {tool: tree['tool']} + concat "".html_safe + concat "
    • ".html_safe if tree.has_key? "tool" + end + +end \ No newline at end of file diff --git a/app/javascript/channels/notification_channel.js b/app/javascript/channels/notification_channel.js new file mode 100644 index 0000000..8855245 --- /dev/null +++ b/app/javascript/channels/notification_channel.js @@ -0,0 +1,17 @@ +import consumer from "./consumer" + +consumer.subscriptions.create("NotificationChannel", { + connected() { + // Called when the subscription is ready for use on the server + console.log("connected") + }, + + disconnected() { + // Called when the subscription has been terminated by the server + }, + + received(data) { + // Called when there's incoming data on the websocket for this channel + console.log("received: ", data) + } +}); diff --git a/app/javascript/packs/controllers/datasets_controller.js b/app/javascript/packs/controllers/datasets_controller.js index 40abca3..76f2817 100644 --- a/app/javascript/packs/controllers/datasets_controller.js +++ b/app/javascript/packs/controllers/datasets_controller.js @@ -16,6 +16,8 @@ export default class extends Controller { $("#dataset-title").val("") modalButton.innerHTML = "Create" modalButton.removeAttribute('disabled') + $(document.body).removeClass("modal-open") + $(".modal-backdrop").remove() }) modal.addEventListener('shown.bs.modal', (event) => { document.getElementById('dataset-title').focus() diff --git a/app/javascript/packs/controllers/experiment_controller.js b/app/javascript/packs/controllers/experiment_controller.js index 42cd024..2736a9e 100644 --- a/app/javascript/packs/controllers/experiment_controller.js +++ b/app/javascript/packs/controllers/experiment_controller.js @@ -5,51 +5,45 @@ import { ServerAPI } from "../utils/server_api" export default class extends Controller { static targets = [] - static values = {lastId: Number, experimentId: Number} + static values = {experimentId: Number} connect() { - this.initPanzoom() + this.panzoom = this.initPanzoom() this.draggable = this.initDraggable() new bootstrap.Offcanvas($("#params_offcanvas")[0]) - this.loadExperiment() } display_tool_config(event) { - let toolSlot = $(event.target).parents(".tool-slot-occupied") - let offcanvasElement = $("#params_offcanvas")[0] - offcanvasElement.innerHTML = Templates.toolParamsOffcanvas(toolSlot.data("tool-params")) - offcanvasElement.setAttribute("data-tool-slot-id", toolSlot.attr('id')) - - let offcanvas = bootstrap.Offcanvas.getInstance(offcanvasElement) - offcanvas.show() + const toolId = $(event.target).closest(".tool-slot-occupied").attr('id').substring(5) + ServerAPI.openToolConfig(toolId, this.experimentIdValue, (data) => { + const offcanvasElement = $("#params_offcanvas")[0] + offcanvasElement.innerHTML = data + offcanvasElement.setAttribute("data-tool-slot-id", toolId) + bootstrap.Offcanvas.getInstance(offcanvasElement).show() + }) } apply_tool_config(event) { // TODO: Check if all config fields are valid - let offcanvasElement = $("#params_offcanvas") - let toolSlot = $(`#${offcanvasElement.attr("data-tool-slot-id")}`) - - const tool_params = toolSlot.data("tool-params") - for(let param of offcanvasElement.find(".tool-param").find("input,select")) { - param = $(param) - const current_param = param.data('param') - tool_params.parameters.filter( (p) => p.name == current_param )[0].value = param.val() - } - toolSlot.data("tool-params", tool_params) - - let current_status = toolSlot.data("status") - toolSlot.data("status","configured") - toolSlot.find("span.tool-status").removeClass(`tool-status-${current_status}`) - toolSlot.find("span.tool-status").addClass(`tool-status-configured`) - this.saveExperiment() - - let offcanvas = bootstrap.Offcanvas.getInstance(offcanvasElement[0]) - offcanvas.hide() + const offcanvasElement = $("#params_offcanvas") + const toolId = offcanvasElement.attr("data-tool-slot-id") + const parameters = {} + $("#params_offcanvas").find(".tool-param").find("input,select").map( (i, e) => { + parameters[e.getAttribute("data-param")] = $(e).val() + }) + ServerAPI.editTool(toolId, parameters, this.experimentIdValue, () => { + bootstrap.Offcanvas.getInstance(offcanvasElement[0]).hide() + }) } delete_tool(event) { - $(event.currentTarget).parents(".tool-slot-occupied").parent().remove() - this.saveExperiment() + const toolId = $(event.target).closest('.tool-slot-occupied').attr('id').substring(5) + ServerAPI.deleteTool(toolId, this.experimentIdValue, (data) => { + this.panzoom.destroy() + this.panzoom = this.initPanzoom() + this.draggable.destroy() + this.draggable = this.initDraggable() + }) } initPanzoom() { @@ -60,6 +54,7 @@ export default class extends Controller { canvas_elem.parentElement.addEventListener('wheel', panzoom.zoomWithWheel) // panzoom.pan(10, 10) // panzoom.zoom(2, { animate: true }) + return panzoom } initDraggable() { @@ -108,21 +103,16 @@ export default class extends Controller { $('html,body').css('cursor','auto') if(final_dropzone != null) { if($(final_dropzone).hasClass("possible-tool-slot")) { - this.lastIdValue++ - const defaultsParams = $(event.originalSource).data('tool') - defaultsParams['toolId'] = this.lastIdValue - final_dropzone.outerHTML = Templates.canvasTool(defaultsParams) - const tool = $(`#tool_${this.lastIdValue}`) - tool.data("tool-params", defaultsParams) - const toolSlot1 = $("
    • ")[0] - const toolSlot2 = $("")[0] - tool[0].parentElement.parentElement.appendChild(toolSlot1) - tool[0].parentElement.appendChild(toolSlot2) - final_dropzone = null - draggable.destroy() - this.draggable.destroy() - this.draggable = this.initDraggable() - this.saveExperiment() + const tool = $(event.originalSource).data('tool') + const parent = final_dropzone.parentElement.parentElement.previousElementSibling + const parentId = (parent == null) ? null : parent.getAttribute('id').substring(5) + ServerAPI.addTool(JSON.stringify(tool), parentId, this.experimentIdValue, (data) => { + final_dropzone = null + this.panzoom.destroy() + this.panzoom = this.initPanzoom() + this.draggable.destroy() + this.draggable = this.initDraggable() + }) } else { $(final_dropzone).html("") @@ -132,62 +122,4 @@ export default class extends Controller { }) return draggable } - - saveExperiment() { - function buildJSON(li) { - let subObj = {} - subObj.tool = li.children().data('tool-params') - li.children('ul').children().each(function() { - const currentLi = $(this) - if (!subObj.children) { - subObj.children = [] - } - if(!currentLi.children().hasClass("tool-slot")) - subObj.children.push(buildJSON(currentLi)) - }) - return subObj - } - const graph = {children: []} - $('ul.tree > li').each(function() { - const currentLi = $(this) - if(!currentLi.children().hasClass("tool-slot")) - graph.children.push(buildJSON(currentLi)) - }) - ServerAPI.saveExperiment(this.experimentIdValue, JSON.stringify(graph), ()=>{}) - } - - loadExperiment() { - function buildList(data, self) { - if(data.tool.toolId > self.lastIdValue) { - self.lastIdValue = data.tool.toolId - } - const li = $("
  • ") - const toolElt = $(Templates.canvasTool(data.tool)) - toolElt.data("tool-params", data.tool) - li.append(toolElt) - const ul = $("") - if(data.children && data.children.length > 0) { - for(const child of data.children) { - ul.append(buildList(child, self)) - } - } - ul.append($("
    • ")) - li.append(ul) - return li - } - - ServerAPI.loadExperiment(this.experimentIdValue, (data) => { - const tree = $("") - if(data.children && data.children.length > 0) { - for(const source_tool of data.children) { - tree.append(buildList(source_tool, this)) - } - } - tree.append($("
    • ")) - $("#experiment_canvas ul.tree").empty() - $("#experiment_canvas ul.tree").append(tree.children()) - this.draggable.destroy() - this.draggable = this.initDraggable() - }) - } } \ No newline at end of file diff --git a/app/javascript/packs/controllers/experiments_controller.js b/app/javascript/packs/controllers/experiments_controller.js new file mode 100644 index 0000000..f62e9f2 --- /dev/null +++ b/app/javascript/packs/controllers/experiments_controller.js @@ -0,0 +1,46 @@ +import { Controller } from "stimulus" +import {ServerAPI} from "../utils/server_api" + +export default class extends Controller { + static targets = [] + static values = {} + + connect() { + this.initModal() + } + + initModal() { + const modal = document.getElementById('createExperimentModal') + const modalButton = document.getElementById('create-experiment-button') + modal.addEventListener('hidden.bs.modal', (event) => { + $("#experiment-title").val("") + modalButton.innerHTML = "Create" + modalButton.removeAttribute('disabled') + $(document.body).removeClass("modal-open") + $(".modal-backdrop").remove() + }) + modal.addEventListener('shown.bs.modal', (event) => { + document.getElementById('experiment-title').focus() + }) + } + + createExperiment(event) { + $("#message").html("") + const title = $("#experiment-title").val() + event.target.setAttribute('disabled', 'disabled') + event.target.innerHTML = `Loading` + ServerAPI.create_experiment(title, (data) => { + if(data['status'] === 'ok') { + //close modal + update list + bootstrap.Modal.getInstance(document.getElementById('createExperimentModal')).hide() + ServerAPI.update_experiments_list((data) => {}) + + } + else { + $("#message").html(data['message']) + event.target.innerHTML = "Create" + event.target.removeAttribute('disabled') + } + }) + } +} \ No newline at end of file diff --git a/app/javascript/packs/controllers/hello_controller.js b/app/javascript/packs/controllers/hello_controller.js deleted file mode 100644 index 5ccd246..0000000 --- a/app/javascript/packs/controllers/hello_controller.js +++ /dev/null @@ -1,10 +0,0 @@ -import { Controller } from "stimulus" - -export default class extends Controller { - connect() { - console.log("Connected.") - } - greet() { - console.log("Hello, Stimulus!", this.element) - } -} \ No newline at end of file diff --git a/app/javascript/packs/utils/server_api.js b/app/javascript/packs/utils/server_api.js index 2d1854d..e82be85 100644 --- a/app/javascript/packs/utils/server_api.js +++ b/app/javascript/packs/utils/server_api.js @@ -1,9 +1,9 @@ export class ServerAPI { - static create_dataset(title, callback) { + static create_experiment(title, callback) { $.ajax({ type: "POST", - url: "/dataset/create", + url: "/experiment/create", data: {title: title}, headers: { 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') @@ -17,10 +17,10 @@ export class ServerAPI { }) } - static update_datasets_list(callback) { + static update_experiments_list(callback) { $.ajax({ type: "GET", - url: "/datasets/update", + url: "/experiments/update", headers: { 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') }, @@ -34,64 +34,134 @@ export class ServerAPI { }) } - static setCurrentWorkingDataset(datasetId, callback) { + static addTool(tool, parentId, experimentId, callback) { $.ajax({ type: "POST", - url: "/datasets/working_dataset", + url: `/experiment/${experimentId}/add_tool`, + data: {tool: tool, parent_id: parentId}, headers: { 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') }, - data: { - dataset_id: datasetId + success: (data, textStatus, jqXHR) => { + callback() + }, + error: (jqXHR, textStatus, errorThrown) => { + + } + }) + } + + static deleteTool(toolId, experimentId, callback) { + $.ajax({ + type: "POST", + url: `/experiment/${experimentId}/delete_tool`, + data: {tool_id: toolId}, + headers: { + 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') + }, + success: (data, textStatus, jqXHR) => { + callback() + }, + error: (jqXHR, textStatus, errorThrown) => { + + } + }) + } + + static openToolConfig(toolId, experimentId, callback) { + $.ajax({ + type: "POST", + url: `/experiment/${experimentId}/edit_tool_form`, + data: {tool_id: toolId}, + headers: { + 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') }, - dataType: "script", success: (data, textStatus, jqXHR) => { callback(data) + }, + error: (jqXHR, textStatus, errorThrown) => { + } }) } - static addSelectedDocumentsToWorkingDataset(documentsIds, callback) { + static editTool(toolId, parameters, experimentId, callback) { $.ajax({ type: "POST", - url: "/datasets/add_documents", + url: `/experiment/${experimentId}/edit_tool`, + data: {tool_id: toolId, parameters: parameters}, headers: { 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') }, - data: { - documents_ids: documentsIds + success: (data, textStatus, jqXHR) => { + callback() + }, + error: (jqXHR, textStatus, errorThrown) => { + + } + }) + } + + static create_dataset(title, callback) { + $.ajax({ + type: "POST", + url: "/dataset/create", + data: {title: title}, + headers: { + 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') + }, + success: (data, textStatus, jqXHR) => { + callback(data) + }, + error: (jqXHR, textStatus, errorThrown) => { + + } + }) + } + + static update_datasets_list(callback) { + $.ajax({ + type: "GET", + url: "/datasets/update", + headers: { + 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') }, dataType: "script", success: (data, textStatus, jqXHR) => { callback(data) + }, + error: (jqXHR, textStatus, errorThrown) => { + } }) } - static paginateDataset(datasetId, page, per_page, sort, sort_order, type, callback) { + static setCurrentWorkingDataset(datasetId, callback) { $.ajax({ type: "POST", - url: `/dataset/${datasetId}/paginate`, + url: "/datasets/working_dataset", headers: { 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') }, data: { - page: page, per_page: per_page, sort: sort, sort_order: sort_order, type: type + dataset_id: datasetId }, - dataType: "script", success: (data, textStatus, jqXHR) => { callback(data) } }) } - static getDatasets(callback) { + static addSelectedDocumentsToWorkingDataset(documentsIds, callback) { $.ajax({ - type: "GET", - url: `/datasets/list`, + type: "POST", + url: "/datasets/add_documents", headers: { 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') }, + data: { + documents_ids: documentsIds + }, dataType: "script", success: (data, textStatus, jqXHR) => { callback(data) @@ -99,29 +169,31 @@ export class ServerAPI { }) } - static saveExperiment(experimentId, experimentGraph, callback) { + static paginateDataset(datasetId, page, per_page, sort, sort_order, type, callback) { $.ajax({ type: "POST", - url: `/experiment/${experimentId}/save`, + url: `/dataset/${datasetId}/paginate`, headers: { 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') }, - data: {description: experimentGraph}, - dataType: "json", + data: { + page: page, per_page: per_page, sort: sort, sort_order: sort_order, type: type + }, + dataType: "script", success: (data, textStatus, jqXHR) => { callback(data) } }) } - static loadExperiment(experimentId, callback) { + static getDatasets(callback) { $.ajax({ type: "GET", - url: `/experiment/${experimentId}/load`, + url: `/datasets/list`, headers: { 'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content') }, - dataType: "json", + dataType: "script", success: (data, textStatus, jqXHR) => { callback(data) } diff --git a/app/javascript/packs/utils/templates.js b/app/javascript/packs/utils/templates.js index 6291cea..82f1e38 100644 --- a/app/javascript/packs/utils/templates.js +++ b/app/javascript/packs/utils/templates.js @@ -9,16 +9,15 @@ export class Templates {
    ${ tool.name } - +
    -
    +
    -
    diff --git a/app/models/experiment.rb b/app/models/experiment.rb index a196a46..b7de592 100644 --- a/app/models/experiment.rb +++ b/app/models/experiment.rb @@ -1,7 +1,85 @@ class Experiment < ActiveRecord::Base belongs_to :user, optional: false + validates :title, length: { minimum: 1 } - + def add_tool(parent_id, tool) + if parent_id != 0 + self.locate_tool(self.description, parent_id) do |t| + t['children'] << tool.to_h + end + else + self.description['children'] << tool.to_h + end + end + + def delete_tool(tool_id) + ids = detach_tool(self.description, nil, tool_id) + puts ids + return ids + end + + def update_tool(tool_id, parameters) + self.locate_tool(self.description, tool_id) do |t| + t['tool']['parameters'].map! do |param| + param['value'] = parameters[param['name']] + param + end + end + end + + private + + def detach_tool(tree, parent_array, tool_id, &block) + if tree.has_key?('tool') + if tree['tool']['id'] == tool_id + ids = gather_ids(tree) + parent_array.delete(tree) unless parent_array.nil? + return ids + else + tree['children'].each do |subtree| + res = detach_tool(subtree, tree['children'], tool_id, &block) + return res unless res.nil? + end + end + else + tree['children'].each do |subtree| + res = detach_tool(subtree, tree['children'], tool_id, &block) + return res unless res.nil? + end + end + nil + end + + def gather_ids(tree, ids=[]) + if tree.has_key?('tool') + ids << tree['tool']['id'] + tree['children'].each do |subtree| + ids.concat(gather_ids(subtree)) + end + end + return ids + end + + def locate_tool(tree_part, parent_id, &block) + if tree_part.has_key?('tool') + if tree_part['tool']['id'] == parent_id + yield tree_part + return true + else + tree_part['children'].each do |subtree| + break if locate_tool(subtree, parent_id, &block) + end + end + else + if tree_part['children'].empty? + yield tree_part + end + tree_part['children'].each do |subtree| + break if locate_tool(subtree, parent_id, &block) + end + end + + end end diff --git a/app/models/tool.rb b/app/models/tool.rb new file mode 100644 index 0000000..9441d1c --- /dev/null +++ b/app/models/tool.rb @@ -0,0 +1,16 @@ +class Tool < ActiveRecord::Base + + belongs_to :experiment, optional: false + + def to_h + { + "tool": { + "id": self.id, + "type": self.tool_type, + "parameters": self.parameters + }, + "children": [] + } + end + +end diff --git a/app/views/experiment/_canvas_tool.html.erb b/app/views/experiment/_canvas_tool.html.erb deleted file mode 100644 index 819cd29..0000000 --- a/app/views/experiment/_canvas_tool.html.erb +++ /dev/null @@ -1,17 +0,0 @@ -
    -
    -
    - <%= tool_data[:name] %> - X -
    -
    -
    -
    <%= tool_data[:title] %>
    -

    blablabla

    -
    -
    - -
    -
    -
    -
    \ No newline at end of file diff --git a/app/views/experiment/_experiment_menu.html.erb b/app/views/experiment/_experiment_menu.html.erb new file mode 100644 index 0000000..215c796 --- /dev/null +++ b/app/views/experiment/_experiment_menu.html.erb @@ -0,0 +1,8 @@ +
    +
    + Experiment +
    +
    +

    <%= @experiment.title %>

    +
    +
    \ No newline at end of file diff --git a/app/views/experiment/_experiments_list.html.erb b/app/views/experiment/_experiments_list.html.erb new file mode 100644 index 0000000..268b63b --- /dev/null +++ b/app/views/experiment/_experiments_list.html.erb @@ -0,0 +1,16 @@ +
    +
    + Experiments +
    +
    +
      + <% current_user.experiments.each do |experiment| %> +
    • + <%= experiment.title %> + Access experiment + 1 +
      • + <% end %> +
    +
    +
    \ No newline at end of file diff --git a/app/views/experiment/_tools_menu.html.erb b/app/views/experiment/_tools_menu.html.erb new file mode 100644 index 0000000..1b6eb32 --- /dev/null +++ b/app/views/experiment/_tools_menu.html.erb @@ -0,0 +1,29 @@ +
    +
    + Tools +
    +
    + +
    + + + +
    +
    +
    \ No newline at end of file diff --git a/app/views/experiment/_tree.html.erb b/app/views/experiment/_tree.html.erb new file mode 100644 index 0000000..d335a1b --- /dev/null +++ b/app/views/experiment/_tree.html.erb @@ -0,0 +1,5 @@ +
    +
      + <% recursive_display(experiment.description) %> +
    +
    \ No newline at end of file diff --git a/app/views/experiment/index.html.erb b/app/views/experiment/index.html.erb index 2628989..ed9d20f 100644 --- a/app/views/experiment/index.html.erb +++ b/app/views/experiment/index.html.erb @@ -1,32 +1,43 @@ <% set_page_title "Experiments" %> -
    +
    Utilities
    - + +
    -
    -
    - Experiments -
    -
    -
      - <% current_user.experiments.each do |experiment| %> -
    • - <%= experiment.id %> - Access experiment - 1 -
      • - <% end %> -
    -
    +
    + <%= render partial: 'experiments_list' %>
    \ No newline at end of file diff --git a/app/views/experiment/show.html.erb b/app/views/experiment/show.html.erb index d16def5..f71cc05 100644 --- a/app/views/experiment/show.html.erb +++ b/app/views/experiment/show.html.erb @@ -2,54 +2,19 @@ <% tools = JSON.parse(@tools_description) %>
    + data-experiment-experiment-id-value="<%= @experiment.id %>">
    -
    -
    -
    - Tools -
    -
    - -
    - - - -
    -
    +
    +
    + <%= render partial: "experiment/experiment_menu" %> +
    +
    + <%= render partial: "experiment/tools_menu", locals: {tools: tools} %>
    -
    -
    -
      -
    • -
      - -
      -
      • -
    -
    + <%= render partial: 'experiment/tree', locals: {experiment: @experiment} %>
    diff --git a/app/views/experiment/update_experiment_area.js.erb b/app/views/experiment/update_experiment_area.js.erb new file mode 100644 index 0000000..d443d65 --- /dev/null +++ b/app/views/experiment/update_experiment_area.js.erb @@ -0,0 +1 @@ +$("#experiment_area").html("<%= j render(partial: "tree", locals: {experiment: @experiment}) %>") \ No newline at end of file diff --git a/app/views/experiment/update_experiments_list.js.erb b/app/views/experiment/update_experiments_list.js.erb new file mode 100644 index 0000000..4cee926 --- /dev/null +++ b/app/views/experiment/update_experiments_list.js.erb @@ -0,0 +1 @@ +$("#experiments_list").html("<%= j render(partial: "experiments_list") %>") \ No newline at end of file diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index 0e0cead..a320cff 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -5,6 +5,7 @@ <%= csrf_meta_tags %> <%= csp_meta_tag %> + <%= action_cable_meta_tag %> <%#= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track': 'reload' %> <%= stylesheet_pack_tag 'application', media: 'all', 'data-turbolinks-track': 'reload' %> diff --git a/app/views/named_entities/_named_entities.html.erb b/app/views/named_entities/_named_entities.html.erb new file mode 100644 index 0000000..e69de29 diff --git a/app/views/tool/_canvas_tool.html.erb b/app/views/tool/_canvas_tool.html.erb new file mode 100644 index 0000000..11cfa3f --- /dev/null +++ b/app/views/tool/_canvas_tool.html.erb @@ -0,0 +1,19 @@ +
    +
    +
    + <%= t("newspapers.tools.type.#{tool['type']}") %> + +
    +
    +
    + +
    +
    +
    +
    \ No newline at end of file diff --git a/app/views/experiment/_menu_tool.html.erb b/app/views/tool/_menu_tool.html.erb similarity index 100% rename from app/views/experiment/_menu_tool.html.erb rename to app/views/tool/_menu_tool.html.erb diff --git a/app/views/tool/_parameters.html.erb b/app/views/tool/_parameters.html.erb new file mode 100644 index 0000000..2c2cdeb --- /dev/null +++ b/app/views/tool/_parameters.html.erb @@ -0,0 +1,37 @@ +
    +
    <%= t("newspapers.tools.type.#{tool.tool_type}") %>
    + +
    +
    + <% tool.parameters.each do |param| %> +
    + <%= param['name'] %> + <% case param['type'] %> + <% when "string" %> + + <% when "float" %> + + <% when "integer" %> + + <% when "select" %> + + <% end %> +
    + <% end %> + +
    \ No newline at end of file diff --git a/app/workers/tool_runner_worker.rb b/app/workers/tool_runner_worker.rb new file mode 100644 index 0000000..52c227e --- /dev/null +++ b/app/workers/tool_runner_worker.rb @@ -0,0 +1,7 @@ +class ToolRunnerWorker + include Sidekiq::Worker + + def perform(user_id) + ActionCable.server.broadcast("notifications.#{user_id}", {test: 'done'}) + end +end diff --git a/config/application.rb b/config/application.rb index 9609f05..66dde98 100644 --- a/config/application.rb +++ b/config/application.rb @@ -7,17 +7,19 @@ require "rails/all" Bundler.require(*Rails.groups) module NewspapersPlatform - class Application < Rails::Application - # Initialize configuration defaults for originally generated Rails version. - config.load_defaults 6.1 + class Application < Rails::Application + # Initialize configuration defaults for originally generated Rails version. + config.load_defaults 6.1 - # Configuration for the application, engines, and railties goes here. - # - # These settings can be overridden in specific environments using the files - # in config/environments, which are processed later. - # - # config.time_zone = "Central Time (US & Canada)" - # config.eager_load_paths << Rails.root.join("extras") - config.solr = config_for('solr') - end + # Configuration for the application, engines, and railties goes here. + # + # These settings can be overridden in specific environments using the files + # in config/environments, which are processed later. + # + # config.time_zone = "Central Time (US & Canada)" + # config.eager_load_paths << Rails.root.join("extras") + config.solr = config_for('solr') + + config.active_job.queue_adapter = :sidekiq + end end diff --git a/config/cable.yml b/config/cable.yml index c721a49..d9a6136 100644 --- a/config/cable.yml +++ b/config/cable.yml @@ -1,5 +1,7 @@ development: - adapter: async + adapter: redis + url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> + channel_prefix: newspapers_platform_production_dev test: adapter: test @@ -7,4 +9,4 @@ test: production: adapter: redis url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> - channel_prefix: newspapers_platform_production + channel_prefix: newspapers_platform_production_prod diff --git a/config/initializers/warden_hooks.rb b/config/initializers/warden_hooks.rb new file mode 100644 index 0000000..a8165d3 --- /dev/null +++ b/config/initializers/warden_hooks.rb @@ -0,0 +1,9 @@ +Warden::Manager.after_set_user do |user, auth, opts| + auth.cookies.signed["user.id"] = user.id + auth.cookies.signed["user.expires_at"] = 30.minutes.from_now +end + +Warden::Manager.before_logout do |user, auth, opts| + auth.cookies.signed["user.id"] = nil + auth.cookies.signed["user.expires_at"] = nil +end \ No newline at end of file diff --git a/config/locales/newspapers.en.yml b/config/locales/newspapers.en.yml index 3207d01..cb1845d 100644 --- a/config/locales/newspapers.en.yml +++ b/config/locales/newspapers.en.yml @@ -25,4 +25,7 @@ en: language: "Language" newspaper: "Newspaper" persons: "Persons" - locations: "Locations" \ No newline at end of file + locations: "Locations" + tools: + type: + source_dataset: "Source Dataset" \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index dc0968c..4348cde 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,25 +1,38 @@ # frozen_string_literal: true - +require 'sidekiq/web' Rails.application.routes.draw do - devise_for :users - # For details on the DSL available within this file, see https://guides.rubyonrails.org/routing.html - root to: 'catalog#home' + devise_for :users + # For details on the DSL available within this file, see https://guides.rubyonrails.org/routing.html + root to: 'catalog#home' + + get '/search', to: 'catalog#index' + get '/catalog/:id', to: 'catalog#show' + post '/paginate_facets', to: 'catalog#paginate_facets' + + get '/datasets', to: 'dataset#index' + get '/datasets/update', to: 'dataset#update_datasets_list' + post '/datasets/working_dataset', to: 'dataset#set_working_dataset' + post "/datasets/add_documents", to: "dataset#add_documents" + get "/datasets/list", to: "dataset#list_datasets" + get '/dataset/:id', to: 'dataset#show' + post "/dataset/:id/paginate", to: "dataset#paginate" + post '/dataset/create', to: 'dataset#create_dataset' - get '/search', to: 'catalog#index' - get '/catalog/:id', to: 'catalog#show' - post '/paginate_facets', to: 'catalog#paginate_facets' + get '/experiments', to: 'experiment#index' + get '/experiments/update', to: 'experiment#update_experiments_list' + post '/experiment/create', to: 'experiment#create' + get '/experiment/:id', to: "experiment#show" + get '/experiment/:id/load', to: "experiment#load" + post '/experiment/:id/save', to: "experiment#save" + post '/experiment/:id/add_tool', to: "experiment#add_tool" + post '/experiment/:id/delete_tool', to: "experiment#delete_tool" + post '/experiment/:id/edit_tool', to: "experiment#edit_tool" + post '/experiment/:id/edit_tool_form', to: "experiment#edit_tool_form" - get '/datasets', to: 'dataset#index' - get '/datasets/update', to: 'dataset#update_datasets_list' - post '/datasets/working_dataset', to: 'dataset#set_working_dataset' - post "/datasets/add_documents", to: "dataset#add_documents" - get "/datasets/list", to: "dataset#list_datasets" - get '/dataset/:id', to: 'dataset#show' - post "/dataset/:id/paginate", to: "dataset#paginate" - post '/dataset/create', to: 'dataset#create_dataset' + resources :tool, only: [:show, :create, :update, :destroy] - get '/experiments', to: 'experiment#index' - get '/experiment/:id', to: "experiment#show" - get '/experiment/:id/load', to: "experiment#load" - post '/experiment/:id/save', to: "experiment#save" + mount ActionCable.server => '/cable' + if Rails.env.development? + mount Sidekiq::Web => '/sidekiq' + end end diff --git a/db/migrate/20210903194218_create_experiment.rb b/db/migrate/20210903194218_create_experiment.rb index d0ed77c..8c0761e 100644 --- a/db/migrate/20210903194218_create_experiment.rb +++ b/db/migrate/20210903194218_create_experiment.rb @@ -3,8 +3,9 @@ class CreateExperiment < ActiveRecord::Migration[6.1] create_table :experiments do |t| t.string :title t.references :user, foreign_key: true - t.jsonb :description + t.jsonb :description, default: {children:[]} t.timestamps end + add_index :experiments, [:title, :user_id], unique: true end end diff --git a/db/migrate/20210915140752_add_tool.rb b/db/migrate/20210915140752_add_tool.rb new file mode 100644 index 0000000..cee1661 --- /dev/null +++ b/db/migrate/20210915140752_add_tool.rb @@ -0,0 +1,12 @@ +class AddTool < ActiveRecord::Migration[6.1] + def change + create_table :tools do |t| + t.references :experiment, foreign_key: true + t.string :tool_type + t.jsonb :parameters, default: {} + t.jsonb :results, default: {} + t.string :status, default: "created" + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index ffe507c..ffa7dc9 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_09_09_142841) do +ActiveRecord::Schema.define(version: 2021_09_15_140752) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -29,12 +29,24 @@ ActiveRecord::Schema.define(version: 2021_09_09_142841) do create_table "experiments", force: :cascade do |t| t.string "title" t.bigint "user_id" - t.jsonb "description" + t.jsonb "description", default: {"children"=>[]} t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false + t.index ["title", "user_id"], name: "index_experiments_on_title_and_user_id", unique: true t.index ["user_id"], name: "index_experiments_on_user_id" end + create_table "tools", force: :cascade do |t| + t.bigint "experiment_id" + t.string "tool_type" + t.jsonb "parameters", default: {} + t.jsonb "results", default: {} + t.string "status", default: "created" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["experiment_id"], name: "index_tools_on_experiment_id" + end + create_table "users", force: :cascade do |t| t.string "email", default: "", null: false t.string "encrypted_password", default: "", null: false @@ -49,4 +61,5 @@ ActiveRecord::Schema.define(version: 2021_09_09_142841) do add_foreign_key "datasets", "users" add_foreign_key "experiments", "users" + add_foreign_key "tools", "experiments" end diff --git a/test/channels/notification_channel_test.rb b/test/channels/notification_channel_test.rb new file mode 100644 index 0000000..a2c6b44 --- /dev/null +++ b/test/channels/notification_channel_test.rb @@ -0,0 +1,8 @@ +require "test_helper" + +class NotificationChannelTest < ActionCable::Channel::TestCase + # test "subscribes" do + # subscribe + # assert subscription.confirmed? + # end +end diff --git a/test/workers/tool_runner_worker_test.rb b/test/workers/tool_runner_worker_test.rb new file mode 100644 index 0000000..29db595 --- /dev/null +++ b/test/workers/tool_runner_worker_test.rb @@ -0,0 +1,6 @@ +require 'test_helper' +class ToolRunnerWorkerTest < Minitest::Test + def test_example + skip "add some examples to (or delete) #{__FILE__}" + end +end -- GitLab