Support importing Wheels by specifying artifact classifiers (#230)

Add new wheel option to get wheel instead of sdist
Support importing wheels by specifying classifier

pivy-importer/build.gradle

@@ -23,6 +23,9 @@ dependencies {
     compile 'ch.qos.logback:logback-classic:1.1.3'
     compile 'org.apache.commons:commons-compress:1.10'
     compile 'commons-io:commons-io:2.4'
+
+    testCompile group: "junit", name: "junit", version: "4.+"
+    testCompile ('org.spockframework:spock-core:1.1-groovy-2.4')
 }
 
 task setupTestingIvyRepo
@@ -82,7 +85,7 @@ task importRequiredDependencies(type: JavaExec) { task ->
 
 license {
     header rootProject.file('codequality/HEADER')
-    exclude '**/*.template'
+    includes(["**/*.java", "**/*.groovy"])
 }
 
 licenseReport {

pivy-importer/src/main/groovy/com/linkedin/python/importer/ImporterCLI.java

@@ -17,6 +17,8 @@
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.Logger;
+import com.linkedin.python.importer.deps.SdistDownloader;
+import com.linkedin.python.importer.deps.WheelsDownloader;
 import com.linkedin.python.importer.deps.DependencyDownloader;
 import com.linkedin.python.importer.deps.DependencySubstitution;
 import org.apache.commons.cli.CommandLine;
@@ -42,7 +44,6 @@ private ImporterCLI() {
     }
 
     public static void main(String[] args) throws Exception {
-
         Options options = createOptions();
 
         CommandLineParser parser = new DefaultParser();
@@ -57,37 +58,64 @@ public static void main(String[] args) throws Exception {
             throw new RuntimeException("Unable to continue, no repository location given on the command line (use the --repo switch)");
         }
         final File repoPath = new File(line.getOptionValue("repo"));
-        final DependencySubstitution replacements = new DependencySubstitution(buildSubstitutionMap(line), buildForceMap(line));
 
         repoPath.mkdirs();
 
         if (!repoPath.exists() || !repoPath.isDirectory()) {
             throw new RuntimeException("Unable to continue, " + repoPath.getAbsolutePath() + " does not exist, or is not a directory");
         }
 
+        importPackages(line, repoPath);
+        logger.info("Execution Finished!");
+    }
 
+    private static void importPackages(CommandLine line, File repoPath) {
+        final DependencySubstitution replacements = new DependencySubstitution(buildSubstitutionMap(line), buildForceMap(line));
         Set<String> processedDependencies = new HashSet<>();
         for (String dependency : line.getArgList()) {
-            DependencyDownloader dependencyDownloader = new DependencyDownloader(
-                    dependency, repoPath, replacements, line.hasOption("latest"), line.hasOption("pre"),
-                    line.hasOption("lenient"));
-            dependencyDownloader.getProcessedDependencies().addAll(processedDependencies);
-            dependencyDownloader.download();
-            processedDependencies.addAll(dependencyDownloader.getProcessedDependencies());
+            DependencyDownloader artifactDownloader;
+
+            if (dependency.split(":").length == 2) {
+                artifactDownloader = new SdistDownloader(dependency, repoPath, replacements, processedDependencies);
+            } else if (dependency.split(":").length == 3) {
+                artifactDownloader = new WheelsDownloader(dependency, repoPath, replacements, processedDependencies);
+            } else {
+                String errMsg = "Unable to parse the dependency "
+                    + dependency
+                    + ".\nThe format of dependency should be either <module>:<revision> for source distribution "
+                    + "or <module>:<revision>:<classifier> for Wheels.";
+
+                if (line.hasOption("lenient")) {
+                    logger.error(errMsg);
+                    continue;
+                }
+                throw new IllegalArgumentException(errMsg);
+            }
+
+            artifactDownloader.download(line.hasOption("latest"), line.hasOption("pre"), line.hasOption("lenient"));
         }
+    }
 
-        logger.info("Execution Finished!");
+    public static void pullDownPackageAndDependencies(Set<String> processedDependencies,
+                                                      DependencyDownloader artifactDownloader,
+                                                      boolean latestVersions,
+                                                      boolean allowPreReleases,
+                                                      boolean lenient) {
+
+        artifactDownloader.getProcessedDependencies().addAll(processedDependencies);
+        artifactDownloader.download(latestVersions, allowPreReleases, lenient);
+        processedDependencies.addAll(artifactDownloader.getProcessedDependencies());
     }
 
     private static Map<String, String> buildForceMap(CommandLine line) {
         Map<String, String> sub = new LinkedHashMap<>();
         if (line.hasOption("force")) {
             for (String it : Arrays.asList(line.getOptionValues("force"))) {
-                System.out.println(it);
                 String[] split = it.split(":");
                 sub.put(split[0], split[1]);
             }
         }
+
         return sub;
     }
 
@@ -154,13 +182,10 @@ public static Map<String, String> buildSubstitutionMap(CommandLine line) {
         Map<String, String> sub = new LinkedHashMap<>();
         if (line.hasOption("replace")) {
             for (String it : Arrays.asList(line.getOptionValues("replace"))) {
-                System.out.println(it);
                 String[] split = it.split("=");
                 sub.put(split[0], split[1]);
             }
         }
         return sub;
-
     }
-
 }

pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/DependencyDownloader.groovy

@@ -15,82 +15,49 @@
  */
 package com.linkedin.python.importer.deps
 
-import com.linkedin.python.importer.distribution.SourceDistPackage
-import com.linkedin.python.importer.ivy.IvyFileWriter
 import com.linkedin.python.importer.pypi.PypiApiCache
 import com.linkedin.python.importer.util.ProxyDetector
 import groovy.util.logging.Slf4j
 import org.apache.commons.io.FilenameUtils
 import org.apache.http.client.fluent.Request
 
-import java.nio.file.Paths
-
 @Slf4j
-class DependencyDownloader {
-
-
+abstract class DependencyDownloader {
     Queue<String> dependencies = [] as Queue
-    Set<String> processedDependencies = [] as Set
     PypiApiCache cache = new PypiApiCache()
+
+    String project
     File ivyRepoRoot
     DependencySubstitution dependencySubstitution
-    boolean latestVersions
-    boolean allowPreReleases
-    boolean lenient
+    Set<String> processedDependencies
 
-    DependencyDownloader(String project, File ivyRepoRoot, DependencySubstitution dependencySubstitution,
-                         boolean latestVersions, boolean allowPreReleases, boolean lenient) {
-        this.dependencySubstitution = dependencySubstitution
+    protected DependencyDownloader(
+        String project,
+        File ivyRepoRoot,
+        DependencySubstitution dependencySubstitution,
+        Set<String> processedDependencies) {
+
+        this.project = project
         this.ivyRepoRoot = ivyRepoRoot
-        this.latestVersions = latestVersions
-        this.allowPreReleases = allowPreReleases
-        this.lenient = lenient
+        this.dependencySubstitution = dependencySubstitution
+        this.processedDependencies = processedDependencies
         dependencies.add(project)
     }
 
-    def download() {
+    def download(boolean latestVersions, boolean allowPreReleases, boolean lenient) {
         while (!dependencies.isEmpty()) {
             def dep = dependencies.poll()
             if (dep in processedDependencies) {
                 continue
             }
-            downloadDependency(dep)
+            downloadDependency(dep, latestVersions, allowPreReleases, lenient)
             processedDependencies.add(dep)
         }
     }
 
-    def downloadDependency(String dep) {
-        log.info("Pulling in $dep")
-        def (name, version) = dep.split(":")
-
-        def projectDetails = cache.getDetails(name)
-        version = projectDetails.maybeFixVersion(version)
-        def sdistDetails = projectDetails.findVersion(version).find { it.packageType == 'sdist' }
-
-        if (sdistDetails == null) {
-            if (lenient) {
-                log.error("Unable to find source dist for $dep")
-                return
-            }
-            throw new RuntimeException("Unable to find source dist for $dep")
-        }
-
-        def destDir = Paths.get(ivyRepoRoot.absolutePath, "pypi", name, version).toFile()
-
-        destDir.mkdirs()
-
-        def artifact = downloadArtifact(destDir, sdistDetails.url)
-        def packageDependencies = new SourceDistPackage(artifact, cache, dependencySubstitution,
-            latestVersions, allowPreReleases).dependencies
-
-        new IvyFileWriter(name, version, [sdistDetails], packageDependencies).writeIvyFile(destDir)
+    abstract downloadDependency(String dep, boolean latestVersions, boolean allowPreReleases, boolean lenient)
 
-        packageDependencies.each { key, value ->
-            dependencies.addAll(value)
-        }
-    }
-
-    static File downloadArtifact(File destDir, String url) {
+    protected static File downloadArtifact(File destDir, String url) {
 
         def filename = FilenameUtils.getName(new URL(url).getPath())
         def contents = new File(destDir, filename)
@@ -118,4 +85,14 @@ class DependencyDownloader {
 
         return contents
     }
+
+    /**
+     * Get the actual module name from artifact name, which has the correct letter case.
+     * @param filename the filename of artifact
+     * @param revision module version
+     * @return actual module name, which is from PyPI
+     */
+    static String getActualModuleNameFromFilename(String filename, String revision) {
+        return filename.substring(0, filename.indexOf(revision) - 1)
+    }
 }

pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/SdistDownloader.groovy

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2016 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.python.importer.deps
+
+import com.linkedin.python.importer.distribution.SourceDistPackage
+import com.linkedin.python.importer.ivy.IvyFileWriter
+import groovy.util.logging.Slf4j
+import java.nio.file.Paths
+
+@Slf4j
+class SdistDownloader extends DependencyDownloader {
+    static final String SOURCE_DIST_PACKAGE_TYPE = "sdist"
+    static final String SOURCE_DIST_ORG = "pypi"
+
+    SdistDownloader(
+        String project,
+        File ivyRepoRoot,
+        DependencySubstitution dependencySubstitution,
+        Set<String> processedDependencies) {
+
+        super(project, ivyRepoRoot, dependencySubstitution, processedDependencies)
+    }
+
+    @Override
+    def downloadDependency(String dep, boolean latestVersions, boolean allowPreReleases, boolean lenient) {
+        def (String name, String version) = dep.split(":")
+
+        def projectDetails = cache.getDetails(name, lenient)
+        // project name is illegal, which means we can't find any information about this project on PyPI
+        if (projectDetails == null) {
+            return
+        }
+
+        version = projectDetails.maybeFixVersion(version)
+        def sdistDetails = projectDetails.findVersion(version).find { it.packageType == SOURCE_DIST_PACKAGE_TYPE }
+
+        if (sdistDetails == null) {
+            if (lenient) {
+                log.error("Unable to find source dist for $dep")
+                return
+            }
+            throw new RuntimeException("Unable to find source dist for $dep")
+        }
+
+        // make sure the module name has the right letter case and dash or underscore as PyPI
+        name = getActualModuleNameFromFilename(sdistDetails.filename, version)
+        log.info("Pulling in $name:$version")
+
+        def destDir = Paths.get(ivyRepoRoot.absolutePath, SOURCE_DIST_ORG, name, version).toFile()
+        destDir.mkdirs()
+
+        def sdistArtifact = downloadArtifact(destDir, sdistDetails.url)
+        def packageDependencies = new SourceDistPackage(name, version, sdistArtifact, cache, dependencySubstitution).getDependencies(latestVersions, allowPreReleases, lenient)
+
+        new IvyFileWriter(name, version, SOURCE_DIST_PACKAGE_TYPE, [sdistDetails]).writeIvyFile(destDir, packageDependencies)
+
+        packageDependencies.each { key, value ->
+            dependencies.addAll(value)
+        }
+    }
+}