Source code

001package org.galaxyproject.dockstore_galaxy_interface.language;
002
003import static org.galaxyproject.gxformat2.Cytoscape.END_ID;
004import static org.galaxyproject.gxformat2.Cytoscape.START_ID;
005
006import com.fasterxml.jackson.databind.ObjectMapper;
007import com.google.common.collect.Lists;
008import com.google.common.collect.Sets;
009import io.dockstore.common.DescriptorLanguage;
010import io.dockstore.common.VersionTypeValidation;
011import io.dockstore.language.CompleteLanguageInterface;
012import io.dockstore.language.RecommendedLanguageInterface;
013import java.net.MalformedURLException;
014import java.net.URL;
015import java.nio.file.Path;
016import java.nio.file.Paths;
017import java.util.HashMap;
018import java.util.List;
019import java.util.Map;
020import java.util.Optional;
021import java.util.Set;
022import java.util.regex.Pattern;
023import java.util.stream.Collectors;
024import org.apache.commons.lang3.ObjectUtils;
025import org.galaxyproject.gxformat2.Cytoscape;
026import org.galaxyproject.gxformat2.Lint;
027import org.galaxyproject.gxformat2.LintContext;
028import org.pf4j.Extension;
029import org.pf4j.Plugin;
030import org.pf4j.PluginWrapper;
031import org.slf4j.Logger;
032import org.slf4j.LoggerFactory;
033import org.yaml.snakeyaml.Yaml;
034import org.yaml.snakeyaml.error.YAMLException;
035
036/** @author jmchilton */
037public class GalaxyWorkflowPlugin extends Plugin {
038  public static final Logger LOG = LoggerFactory.getLogger(GalaxyWorkflowPlugin.class);
039  public static final String[] TEST_SUFFIXES = {"-tests", "_tests", "-test", "-tests"};
040  public static final String[] TEST_EXTENSIONS = {".yml", ".yaml", ".json"};
041
042  /**
043   * Constructor to be used by plugin manager for plugin instantiation. Your plugins have to provide
044   * constructor with this exact signature to be successfully loaded by manager.
045   *
046   * @param wrapper
047   */
048  public GalaxyWorkflowPlugin(PluginWrapper wrapper) {
049    super(wrapper);
050  }
051
052  @Extension
053  public static class GalaxyWorkflowPluginImpl implements CompleteLanguageInterface {
054    private ObjectMapper mapper = new ObjectMapper();
055
056    /**
057     * This is basically stolen from org.galaxyproject.gxformat2.Lint. However, that is generated
058     * code and cannot be modified so putting it here along with this warning that the two should
059     * probably be in sync.
060     *
061     * @param workflow
062     * @return
063     */
064    public static boolean isGXFormat2Workflow(final Map<String, Object> workflow) {
065      final String wfClass = (String) workflow.get("class");
066      return "GalaxyWorkflow".equals(wfClass);
067    }
068
069    @Override
070    public String launchInstructions(String trsID) {
071      return null;
072    }
073
074    @Override
075    public Map<String, Object> loadCytoscapeElements(
076        String initialPath, String contents, Map<String, FileMetadata> indexedFiles) {
077      final Map<String, Object> workflow = loadWorkflow(contents);
078      try {
079        return Cytoscape.getElements(workflow);
080      } catch (ClassCastException e) {
081        LOG.error(
082            "ClassCastException, looks like an invalid workflow that passed the linter: "
083                + e.getMessage());
084        return Map.of();
085      }
086    }
087
088    @Override
089    public List<RowData> generateToolsTable(
090        String initialPath, String contents, Map<String, FileMetadata> indexedFiles) {
091      final Map<String, Object> workflow = loadWorkflow(contents);
092      final Map<String, Object> elements = Cytoscape.getElements(workflow);
093      final List<Map> nodes = (List<Map>) elements.getOrDefault("nodes", Lists.newArrayList());
094      removeStartAndEndNodes(nodes);
095      return nodes.stream()
096          .map(
097              node -> {
098                final RowData rowData = new RowData();
099                final Map<String, Object> nodeData = (Map<String, Object>) node.get("data");
100                rowData.label = (String) nodeData.getOrDefault("label", "");
101                rowData.dockerContainer = (String) nodeData.getOrDefault("docker", "");
102                rowData.filename = (String) nodeData.getOrDefault("run", "");
103                // TODO: get a sane link here when Docker is hooked up
104                try {
105                  rowData.link = new URL((String) node.getOrDefault("repo_link", ""));
106                } catch (MalformedURLException e) {
107                  rowData.link = null;
108                }
109                rowData.rowType = RowType.TOOL;
110                rowData.toolid = (String) nodeData.getOrDefault("id", "");
111                return rowData;
112              })
113          .collect(Collectors.toList());
114    }
115
116    // Remove start and end nodes that were added for DAG
117    private void removeStartAndEndNodes(List<Map> nodes) {
118      nodes.removeIf(
119          node -> {
120            Map data = (Map) node.get("data");
121            String id = (String) data.get("id");
122            return START_ID.equals(id) || END_ID.equals(id);
123          });
124    }
125
126    @Override
127    public VersionTypeValidation validateWorkflowSet(
128        String initialPath, String contents, Map<String, FileMetadata> indexedFiles) {
129      final LintContext lintContext = Lint.lint(loadWorkflow(contents));
130      final boolean valid;
131      valid = !lintContext.getFoundErrors();
132      final Map<String, String> messagesAsMap = new HashMap<>();
133      final List<String> validationMessages = lintContext.collectMessages();
134      final StringBuilder builder = new StringBuilder();
135      if (validationMessages.size() == 1) {
136        builder.append(validationMessages.get(0));
137      } else if (validationMessages.size() > 1) {
138        for (final String validationMessage : validationMessages) {
139          builder.append("- ").append(validationMessage).append("\n");
140        }
141      }
142      final String validationMessageMerged = builder.toString();
143      if (validationMessageMerged.length() > 0) {
144        messagesAsMap.put(initialPath, validationMessageMerged);
145      }
146      return new VersionTypeValidation(valid, messagesAsMap);
147    }
148
149    @Override
150    public VersionTypeValidation validateTestParameterSet(Map<String, FileMetadata> indexedFiles) {
151      return new VersionTypeValidation(true, new HashMap<>());
152    }
153
154    @Override
155    public DescriptorLanguage getDescriptorLanguage() {
156      return DescriptorLanguage.GXFORMAT2;
157    }
158
159    @Override
160    public Pattern initialPathPattern() {
161      // Why doesn't this seem to be called anywhere?
162      return Pattern.compile("/(.*\\.gxwf\\.y[a]?ml|.*\\.ga)");
163    }
164
165    @Override
166    public Map<String, FileMetadata> indexWorkflowFiles(
167        final String initialPath, final String contents, final FileReader reader) {
168      Map<String, FileMetadata> results = new HashMap<>();
169
170      // identify filetype of initial descriptor, copied from Lint.java
171      String languageVersion = null;
172      try {
173        final Map<String, Object> workflowMap = loadWorkflow(contents);
174        if (isGXFormat2Workflow(workflowMap)) {
175          languageVersion = "gxformat2";
176        } else {
177          languageVersion = "gxformat1";
178        }
179      } catch (Exception e) {
180        // do nothing
181      }
182
183      results.put(
184          initialPath,
185          new FileMetadata(contents, GenericFileType.IMPORTED_DESCRIPTOR, languageVersion));
186
187      final Optional<String> testParameterFile = findTestParameterFile(initialPath, reader);
188      testParameterFile.ifPresent(
189          // TODO - get language version into here
190          s -> results.put(s, new FileMetadata(s, GenericFileType.TEST_PARAMETER_FILE, null)));
191      return results;
192    }
193
194    protected Optional<String> findTestParameterFile(
195        final String initialPath, final FileReader reader) {
196      final int extensionPos = initialPath.lastIndexOf(".");
197      final String base = initialPath.substring(0, extensionPos);
198
199      final Path parent = Paths.get(initialPath).getParent();
200      // listing files is more rate limit friendly (e.g. GitHub counts each 404 "miss" as an API
201      // call,
202      // but listing a directory can be free if previously requested/cached)
203      final Set<String> filenameset =
204          parent == null ? Sets.newHashSet() : Sets.newHashSet(reader.listFiles(parent.toString()));
205
206      for (final String suffix : TEST_SUFFIXES) {
207        for (final String extension : TEST_EXTENSIONS) {
208          final String testFile = base + suffix + extension;
209          if (filenameset.contains(testFile)) {
210            return Optional.of(testFile);
211          }
212        }
213      }
214      return Optional.empty();
215    }
216
217    protected Boolean pathExistsFromReader(final FileReader reader, final String path) {
218      try {
219        reader.readFile(path);
220        return true;
221      } catch (Exception e) {
222        return false;
223      }
224    }
225
226    static Map<String, Object> loadWorkflow(final String content) {
227      final Yaml yaml = new Yaml();
228      final Map map = yaml.loadAs(content, Map.class);
229      return (Map<String, Object>) map;
230    }
231
232    @Override
233    public RecommendedLanguageInterface.WorkflowMetadata parseWorkflowForMetadata(
234        String initialPath, String content, Map<String, FileMetadata> indexedFiles) {
235      RecommendedLanguageInterface.WorkflowMetadata metadata =
236          new RecommendedLanguageInterface.WorkflowMetadata();
237      if (content != null && !content.isEmpty()) {
238        try {
239          final Map<String, Object> map = loadWorkflow(content);
240          String name = null;
241          try {
242            name = (String) map.get("name");
243          } catch (ClassCastException e) {
244            LOG.debug("\"name:\" is malformed");
245          }
246
247          // FOLLOWING CODE PULLED FROM cwl handler...
248          String label = null;
249          try {
250            label = (String) map.get("label");
251          } catch (ClassCastException e) {
252            LOG.debug("\"label:\" is malformed");
253          }
254
255          String annotation = null;
256          try {
257            annotation = (String) map.get("annotation");
258          } catch (ClassCastException e) {
259            LOG.debug("\"annotation:\" is malformed");
260          }
261
262          // "doc:" added for CWL 1.0
263          String doc = null;
264          if (map.containsKey("doc")) {
265            Object objectDoc = map.get("doc");
266            if (objectDoc instanceof String) {
267              doc = (String) objectDoc;
268            } else if (objectDoc instanceof Map) {
269              Map docMap = (Map) objectDoc;
270              if (docMap.containsKey("$include")) {
271                String enclosingFile = (String) docMap.get("$include");
272                Optional<FileMetadata> first =
273                    indexedFiles.values().stream()
274                        .filter(fileMetadata -> fileMetadata.content().equals(enclosingFile))
275                        .findFirst();
276                if (first.isPresent()) {
277                  // No way to fetch this here...
278                  LOG.info(
279                      "$include would have this but reader not passed through, not implemented");
280                  doc = null;
281                }
282              }
283            } else if (objectDoc instanceof List) {
284              // arrays for "doc:" added in CWL 1.1
285              List docList = (List) objectDoc;
286              doc = String.join(System.getProperty("line.separator"), docList);
287            }
288          }
289
290          final String finalChoiceForDescription =
291              ObjectUtils.firstNonNull(doc, annotation, name, label);
292          if (finalChoiceForDescription != null) {
293            metadata.setDescription(finalChoiceForDescription);
294          } else {
295            LOG.info("Description not found!");
296          }
297
298        } catch (YAMLException | NullPointerException | ClassCastException ex) {
299          String message;
300          if (ex.getCause() != null) {
301            // seems to be possible to get underlying cause in some cases
302            message = ex.getCause().toString();
303          } else {
304            // in other cases, the above will NullPointer
305            message = ex.toString();
306          }
307          LOG.info("Galaxy Workflow file is malformed " + message);
308          // CWL parser gets to put validation information in here,
309          // plugin interface doesn't consume the right information though.
310          // https://github.com/dockstore/dockstore/blob/develop/dockstore-webservice/src/main/java/io/dockstore/webservice/languages/CWLHandler.java#L139
311        }
312      }
313      return metadata;
314    }
315  }
316}