Merge pull request #5515 from galaxyproject/manticore-bowfin

improved by-tools page

.github/workflows/deploy.yml

@@ -70,6 +70,7 @@ jobs:
           make annotate ACTIVATE_ENV=pwd
           curl -L https://docs.google.com/spreadsheets/d/1NfZhi5Jav7kl9zFCkeb7rIC2F8xW1isruv1TeO4WpNI/export\?format\=tsv | ruby bin/prepare_feedback.rb
           curl -L https://hexylena.github.io/toolshed-version-database/tool-meta.json > metadata/tool-meta.json
+          curl -L https://edamontology.org/EDAM.csv > metadata/EDAM.csv
           make rebuild-search-index ACTIVATE_ENV=pwd
           cat metadata/swagger.yaml | python bin/yaml2json.py > api/swagger.json
           rdoc bin _plugins/ --output gtn_rdoc

_includes/cyoa-choices.html

@@ -1,6 +1,8 @@
 <blockquote class="notranslate hands_on gtn-cyoa" id="gtn-cyoa{{ include.disambiguation }}">
-	<div class="box-title" aria-label="hands-on box: choose your own tutorial"><i class="fas fa-pencil-alt" aria-hidden="true"></i><span class="visually-hidden"></span> Hands-on: Choose Your Own Tutorial</div>
+	<div class="box-title" aria-label="hands-on box: choose your own tutorial"><i class="fas fa-pencil-alt" aria-hidden="true"></i><span class="visually-hidden"></span> {{ include.title | default: "Hands-on: Choose Your Own Tutorial" }}</div>
+	{% unless include.brief %}
 	<p>This is a "Choose Your Own Tutorial" section, where you can select between multiple paths. Click one of the buttons below to select how you want to follow the tutorial</p>
+	{% endunless %}
 	{% if include.text %}
 	<p>
 		{{ include.text }}

_layouts/by_tool.html

@@ -2,60 +2,103 @@
 layout: base
 ---
 
-{% assign tool_id_path = page.observed_tool_ids[0][0] %}
+{% assign tool_id_path = page.latest_tool_id %}
 
 <hgroup>
 	<h1>
 		{{ site.data['tool-meta'][tool_id_path].name }}:
 		{{ site.data['tool-meta'][tool_id_path].desc }}
 	</h1>
-	<pre>{{ page.observed_tool_ids[0] }}</pre>
+	<pre>{{ tool_id_path }}</pre>
 </hgroup>
 
 <section>
 	<div class="stats">
 		<div class="row">
 			<div class="col-md-3">
 				<h2>Metadata</h2>
-				<h3>Servers Offering this Tool</h3>
-				<a href="https://usegalaxy.eu/?tool_id={{ page.observed_tool_ids[0][0] }}">UseGalaxy.eu ({{ page.observed_tool_ids[0][1] }})</a>
-				<p class="text-muted">This is a guess, we are not currently checking if that is actually present.</p>
 
-				<hgroup>
-					<h3>Observed Tool Versions</h3>
-					<p>Within GTN tutorials</p>
-				</hgroup>
+				<h3>Links</h3>
 				<ul>
-					{% for tool in page.observed_tool_ids %}
-					<li>{{ tool[1] }}</li>
-					{% endfor %}
+					{% if site.data['tool-meta'][tool_id_path]['bio.tools'] %}
+					<li>
+						<a href="https://bio.tools/{{ site.data['tool-meta'][tool_id_path]['bio.tools'] }}">
+						bio.tools: {{ site.data['tool-meta'][tool_id_path]['bio.tools_name'] }}
+						</a>
+					</li>
+					{% endif %}
+					{% if site.data['toolshed-revisions'][tool_id] %}
+					<li>
+						{% assign repo_info = site.data['toolshed-revisions'][tool_id] %}
+						<a href="https://toolshed.g2.bx.psu.edu/view/{{ repo_info[0] }}/{{ repo_info[1] }}/{{ repo_info[2] }}">
+						Galaxy ToolShed Repository
+						</a>
+					</li>
+					{% endif %}
 				</ul>
 
-				{% if site.data['tool-meta'][tool_id_path].edam_operations %}
+				{% assign edam_op_count = site.data['tool-meta'][tool_id_path].edam_operations | size %}
+				{% if edam_op_count > 0 %}
 				<h3>EDAM Operations</h3>
 				<ul>
 				{% for topic in site.data['tool-meta'][tool_id_path].edam_operations %}
 					<li>
+						{% assign edam_info = topic | edamify: site %}
 						<a href="https://www.ebi.ac.uk/ols4/ontologies/edam/classes/http%253A%252F%252Fedamontology.org%252F{{ topic }}">
-						{{ topic }}
+							{{ edam_info['Preferred Label'] | default: topic }}
 						</a>
 					</li>
 				{% endfor %}
 				</ul>
 				{% endif %}
 
-				{% if site.data['tool-meta'][tool_id_path].edam_topics%}
+				{% assign edam_topic_count = site.data['tool-meta'][tool_id_path].edam_topics | size %}
+				{% if edam_topic_count > 0 %}
 				<h3>EDAM Topics</h3>
 				<ul>
 				{% for topic in site.data['tool-meta'][tool_id_path].edam_topics %}
 					<li>
+						{% assign edam_info = topic | edamify: site %}
 						<a href="https://www.ebi.ac.uk/ols4/ontologies/edam/classes/http%253A%252F%252Fedamontology.org%252F{{ topic }}">
-						{{ topic }}
+							{{ edam_info['Preferred Label'] | default: topic }}
 						</a>
 					</li>
 				{% endfor %}
 				</ul>
 				{% endif %}
+
+				<hgroup>
+					<h3>Observed Tool Versions</h3>
+					<p>Within GTN tutorials</p>
+				</hgroup>
+
+				{% for tool in page.observed_tool_ids %}
+				<details>
+					<summary>{{ tool[1] }}</summary>
+					{% assign v_id = tool[0] %}
+					{% assign supported = site | tool_version_support: v_id %}
+					{% assign usegalaxy_exact = supported.exact | where: "usegalaxy", "true"   %}
+					{% assign other_exact = supported.exact | where: "usegalaxy", "false"   %}
+
+					<ul>
+						{% for inst in usegalaxy_exact %}
+						<li>
+							<a href="{{ inst.url }}?tool_id={{ tool[0] }}&version={{ tool[1] }}">
+								{{ inst.name }} <abbr title="This is a UseGalaxy.* server which meets minimum requirements for a public Galaxy">⭐️</abbr>
+							</a>
+						</li>
+						{% endfor %}
+						{% for inst in other_exact %}
+						<li>
+								<a href="{{ inst.url }}?tool_id={{ tool[0] }}&version={{ tool[1] }}">
+								{{ inst.name }}
+							</a>
+						</li>
+						{% endfor %}
+					</ul>
+				</details>
+				{% endfor %}
+
 			</div>
 
 			<div class="col-md-9">
@@ -74,3 +117,21 @@ <h2>Relevant Tutorials</h2>
 		</div>
 	</div>
 </section>
+
+<style>
+details {
+	border: 1px solid black;
+	margin-bottom: 0.2rem;
+	cursor: pointer;
+
+	summary {
+		padding: 0.5rem
+	}
+}
+details:hover {
+	box-shadow: 5px 5px var(--brand-color)
+}
+details:active {
+	box-shadow: 3px 3px var(--brand-color)
+}
+</style>

_plugins/api.rb

@@ -6,8 +6,10 @@
 require './_plugins/gtn/metrics'
 require './_plugins/gtn/scholar'
 require './_plugins/gtn/git'
+require './_plugins/gtn/hooks'
 require './_plugins/gtn/ro-crate'
 require './_plugins/gtn'
+require './_plugins/util'
 
 ##
 # Use Jekyll's Markdown converter to convert text to HTML
@@ -300,17 +302,6 @@ def generate(site)
       page2.data['layout'] = nil
       site.pages << page2
 
-      # Not really an API
-      TopicFilter.list_materials_by_tool(site).each do |tool, tutorials|
-        page2 = PageWithoutAFile.new(site, '', 'by-tool/', "#{tool.gsub('%20', ' ')}.html")
-        page2.content = nil
-        page2.data['layout'] = 'by_tool'
-        page2.data['short_tool'] = tool
-        page2.data['observed_tool_ids'] = tutorials['tool_id']
-        page2.data['tutorial_list'] = tutorials['tutorials']
-        site.pages << page2
-      end
-
       # GA4GH TRS Endpoint
       # Please note that this is all a fun hack
       TopicFilter.list_all_materials(site).select { |m| m['workflows'] }.each do |material|
@@ -348,6 +339,11 @@ def generate(site)
   end
 end
 
+
+Jekyll::Hooks.register :site, :post_read do |site|
+  Gtn::Hooks.by_tool(site)
+end
+
 # Basically like `PageWithoutAFile`, we just write out the ones we'd created earlier.
 Jekyll::Hooks.register :site, :post_write do |site|
   # No need to run this except in prod.

_plugins/gtn.rb

@@ -342,6 +342,13 @@ def regex_replace_once(str, regex_search, value_replace)
       str.sub(regex, value_replace)
     end
 
+    ##
+    # Check if a match is found
+    def matches(str, regex_search)
+      r = /#{regex_search}/
+      str.match?(r)
+    end
+
     def convert_to_material_list(site, materials)
       # [{"name"=>"introduction", "topic"=>"admin"}]
       return [] if materials.nil?

_plugins/gtn/hooks.rb

@@ -0,0 +1,72 @@
+require './_plugins/jekyll-topic-filter'
+require 'jekyll'
+
+
+module Gtn
+  # Parse the git repo to get some facts
+  module Hooks
+
+    ##
+    # Generate the by-tool pages
+    # Params:
+    # +site+:: Jekyll site object
+    def self.by_tool(site)
+      Jekyll.logger.debug "[GTN/Hooks/by_tool] Started"
+      init_count = site.pages.size
+      start_time = Time.now
+
+      tools = TopicFilter.list_materials_by_tool(site)
+      tools.reject!{|tool, _| tool.include?('{{')}
+
+      tools.each do |tool, tutorials|
+        # tool: e.g. `saskia-hiltemann/krona_text/krona-text`
+
+        ordered_tool_ids = tutorials['tool_id']
+          .map{|x| 
+            if x[0] == x[1]
+              # TODO: collect versions of builtins.
+              [x[0], '0.0.0'] # Fake version for local only tools
+            else
+              x
+            end
+          }
+          .reject{|x| x[0] == x[1]}
+          .map{|x| [x[0], x[1], Gem::Version.new(fix_version(x[1]))]}
+          .sort_by{|x| x[2]}
+
+        # Redirect from the older, shorter IDs that have more potential for conflicts.
+        if tool.include?('/')
+          previous_id = tool.split('/')[0] + '/' + tool.split('/')[2]
+        else
+          previous_id = tool # No change
+        end
+
+        page2 = Jekyll::PageWithoutAFile.new(site, '', 'by-tool/', "#{tool.gsub('%20', ' ')}.html")
+        page2.content = nil
+        page2.data['layout'] = 'by_tool'
+        page2.data['short_tool'] = tool
+        page2.data['observed_tool_ids'] = ordered_tool_ids.map{|x| x[0..1]}.reverse
+        page2.data['tutorial_list'] = tutorials['tutorials']
+        page2.data['latest_tool_id'] = ordered_tool_ids.map{|x| x[0]}.last
+        # page2.data['redirect_from'] = ["/by-tool/#{previous_id.gsub('%20', ' ')}"]
+        site.pages << page2
+
+        # TODO: For whatever reason the redirect_from does NOT work, even this
+        # early in the hooks, so we're just going to write the file and call it
+        # a day. Someone should fix this someday. My apologies for leaving it like this.
+        if previous_id != tool
+          page2 = Jekyll::PageWithoutAFile.new(site, '', 'by-tool/', "#{previous_id}.html")
+          page2.content = nil
+          page2.data['layout'] = 'by_tool'
+          page2.data['short_tool'] = tool
+          page2.data['observed_tool_ids'] = ordered_tool_ids.map{|x| x[0..1]}.reverse
+          page2.data['tutorial_list'] = tutorials['tutorials']
+          page2.data['latest_tool_id'] = ordered_tool_ids.map{|x| x[0]}.last
+          site.pages << page2
+        end
+
+      end
+      Jekyll.logger.info "[GTN/Hooks/by_tool] #{site.pages.size - init_count} pages added in #{Time.now - start_time}s"
+    end
+  end
+end

_plugins/gtn/supported.rb

@@ -72,17 +72,17 @@ def self.calculate(data, tool_list)
       # generate a 'false' value when merging sets.
       inexact_support -= exact_support
 
-      usegalaxy_server_urls = Gtn::Usegalaxy.servers.map { |x| x[:url] }
+      usegalaxy_server_urls = Gtn::Usegalaxy.servers.map { |x| x[:url].downcase.gsub(/\/$/, '')}
 
       {
         'exact' => (exact_support || []).map do |id|
           data['servers'][id].update(
-            { 'usegalaxy' => usegalaxy_server_urls.include?(data['servers'][id]['url']) }
+            { 'usegalaxy' => usegalaxy_server_urls.include?(data['servers'][id]['url'].downcase.gsub(/\/$/, '')) }
           )
         end,
         'inexact' => (inexact_support || []).map do |id|
           data['servers'][id].update(
-            { 'usegalaxy' => usegalaxy_server_urls.include?(data['servers'][id]['url']) }
+            { 'usegalaxy' => usegalaxy_server_urls.include?(data['servers'][id]['url'].downcase.gsub(/\/$/, '')) }
           )
         end
       }

_plugins/jekyll-jsonld.rb

@@ -937,6 +937,35 @@ def generate_material_jsonld(material, topic, site)
         end
       end
 
+      # Tools
+      uses_tools = false
+      (actual_material['tools'] || []).each do |tool|
+        if site.data['tool-meta'].nil?
+          next
+        end
+
+        toolmeta = site.data['tool-meta'][tool]
+        if toolmeta.nil?
+          next
+        end
+
+        if toolmeta['bio.tools'].length.positive?
+          mentions.push({
+                          '@type': 'Thing',
+                          url: "https://bio.tools/tool/#{toolmeta['bio.tools']}",
+                          name: toolmeta.fetch('bio.tools_name', toolmeta['name'])
+                        })
+        end
+        uses_tools = true
+      end
+      if uses_tools
+        mentions.push({
+                        '@type': 'Thing',
+                        url: "https://bio.tools/tool/galaxy",
+                        name: "Galaxy"
+                      })
+      end
+
       # Zenodo link out
       if actual_material.key?('zenodo_link') && ! actual_material['zenodo_link'].nil?
         if actual_material['zenodo_link'].length.positive?

_plugins/jekyll-topic-filter.rb

@@ -952,7 +952,7 @@ def self.get_version(tool)
   # short_tool("toolshed.g2.bx.psu.edu/repos/galaxyp/regex_find_replace/regex1/1.0.0") => "galaxyp/regex1"
   def self.short_tool(tool)
     if tool.count('/') > 4
-      "#{tool.split('/')[2]}/#{tool.split('/')[4]}"
+      "#{tool.split('/')[2]}/#{tool.split('/')[3]}/#{tool.split('/')[4]}"
     else
       tool
     end
@@ -1205,6 +1205,14 @@ def get_workflow(site, page, workflow)
       mat = to_material(site, page)
       mat['workflows'].select { |w| w['workflow'] == workflow }[0]
     end
+
+    def tool_version_support(site, tool)
+      Gtn::Supported.calculate(site.data['public-server-tools'], [tool])
+    end
+
+    def edamify(term, site)
+      site.data['EDAM'].select{|row| row['Class ID'] == "http://edamontology.org/#{term}"}.first.to_h
+    end
   end
 end