Revert "Jgit import"

.gitignore

@@ -1,6 +1,12 @@
 /target
+/lib
 /classes
-.cpcache/
+/checkouts
+/tmp
 pom.xml
 *.jar
 *.class
+.lein-deps-sum
+.lein-failures
+.lein-plugins
+*.tar

README.md

@@ -4,22 +4,17 @@
 
 ## Usage
 
-Clone the **codeq** repo. Then, with the [Clojure CLI](https://clojure.org/guides/install_clojure) installed:
+Clone the **codeq** repo. Then (in it) run:
 
-Build a standalone jar:
+    lein uberjar
 
-    clojure -T:build uber
+Get [Datomic Free](http://www.datomic.com/get-datomic.html)
 
-`codeq` uses [Datomic Pro](https://www.datomic.com/) (free, no license required) via the Peer API. For a quick run with no transactor, use the in-memory `datomic:mem://` storage:
+Unzip it, then start the Datomic Free transactor. Follow the instructions for [running the transactor with the free storage protocol](http://docs.datomic.com/getting-started.html)
 
     cd theGitRepoYouWantToImport
-    java -jar whereverYouPutCodeq/target/codeq.jar datomic:mem://git
 
-For development without building a jar:
-
-    clojure -M:run datomic:mem://git
-
-For persistent storage, start a Datomic Pro transactor and pass its URI (e.g. `datomic:dev://localhost:4334/git`) instead of `datomic:mem://`.
+    java -server -Xmx1g -jar whereverYouPutCodeq/target/codeq-0.1.0-SNAPSHOT-standalone.jar datomic:free://localhost:4334/git
 
 This will create a db called `git` (you can call it whatever you like) and import the commits from the local view of the repo. You should see output like:
 
@@ -43,7 +38,12 @@ The import is not too peppy, since it shells to `git` relentlessly, but it impor
 
 You can import more than one repo into the same db. You can re-import later after some more commits and they will be incrementally added.
 
-You can then (or during) connect to the same db URI with a peer. Note that the in-memory `datomic:mem://` db only lives inside the importing process — to browse a db after import, run against a persistent storage URI backed by a Datomic Pro transactor (e.g. `datomic:dev://localhost:4334/git`).
+You can then (or during) connect to the same db URI with a peer. Or, just start the [Datomic REST service](http://docs.datomic.com/rest.html) and poke around:
+
+    cd whereverYouPutDatomicFree
+    bin/rest -p 8080 free datomic:free://localhost:4334/
+
+Browse to [localhost:8080/data/](http://localhost:8080/data/). You should see the `free` storage and the `git` db within it.
 
 The [schema diagram](https://github.com/downloads/Datomic/codeq/codeq.pdf) will help you get oriented.
 

project.clj

@@ -0,0 +1,11 @@
+(defproject datomic/codeq "0.1.0-SNAPSHOT"
+  :description "codeq does a code-aware import of your git repo into a Datomic db"
+  :url "http://datomic.com"
+  :license {:name "Eclipse Public License"
+            :url "http://www.eclipse.org/legal/epl-v10.html"}
+  :main datomic.codeq.core
+  :plugins [[lein-tar "1.1.0"]]
+  :dependencies [[com.datomic/datomic-free "0.9.4699"]
+                 [commons-codec "1.7"]
+                 [org.clojure/clojure "1.5.1"]]
+  :source-paths ["src" "examples/src"])

src/datomic/codeq/analyzer.clj

@@ -7,7 +7,8 @@
 ;;   You must not remove this notice, or any other, from this software.
 
 (ns datomic.codeq.analyzer
-  (:import [java.io StringReader]))
+  (:import [java.io StringReader]
+           [org.apache.commons.codec.digest DigestUtils]))
 
 (set! *warn-on-reflection* true)
 
@@ -21,9 +22,7 @@
 (defn sha
   "Returns the hex string of the sha1 of s"
   [^String s]
-  (let [bytes (.digest (java.security.MessageDigest/getInstance "SHA-1")
-                       (.getBytes s "UTF-8"))]
-    (apply str (map #(format "%02x" %) bytes))))
+  (org.apache.commons.codec.digest.DigestUtils/shaHex s))
 
 (defn ws-minify
   "Consecutive ws becomes a single space, then trim"

src/datomic/codeq/core.clj

@@ -8,12 +8,13 @@
 
 (ns datomic.codeq.core
   (:require [datomic.api :as d]
+            [clojure.java.io :as io]
             [clojure.set]
             [clojure.string :as string]
             [datomic.codeq.util :refer [index->id-fn tempid?]]
-            [datomic.codeq.git :as git]
             [datomic.codeq.analyzer :as az]
             [datomic.codeq.analyzers.clj])
+  (:import java.util.Date)
   (:gen-class))
 
 (set! *warn-on-reflection* true)
@@ -249,6 +250,13 @@
        :db.install/_attribute :db.part/db}
       ])
 
+(defn ^java.io.Reader exec-stream
+  [^String cmd]
+  (-> (Runtime/getRuntime)
+      (.exec cmd)
+      .getInputStream
+      io/reader))
+
 (defn ensure-schema [conn]
   (or (-> conn d/db (d/entid :tx/commit))
       @(d/transact conn schema)))
@@ -285,11 +293,63 @@
 ;;  Push  URL: https://github.com/Datomic/codeq.git
 ;;  HEAD branch: (not queried)
 
+(defn get-repo-uri
+  "returns [uri name]"
+  []
+  (with-open [s (exec-stream (str "git remote show -n origin"))]
+    (let [es (line-seq s)
+          ^String line (second es)
+          uri (subs line (inc (.lastIndexOf line " ")))
+          noff (.lastIndexOf uri "/")
+          noff (if (not (pos? noff)) (.lastIndexOf uri ":") noff)
+          name (subs uri (inc noff))
+          _ (assert (pos? (count name)) "Can't find remote origin")
+          name (if (.endsWith name ".git") (subs name 0 (.indexOf name ".")) name)]
+      [uri name])))
+
+(defn dir
+  "Returns [[sha :type filename] ...]"
+  [tree]
+  (with-open [s (exec-stream (str "git cat-file -p " tree))]
+    (let [es (line-seq s)]
+      (mapv #(let [ss (string/split ^String % #"\s")]
+               [(nth ss 2)
+                (keyword (nth ss 1))
+                (subs % (inc (.indexOf ^String % "\t")) (count %))])
+            es))))
+
+(defn commit
+  [[sha _]]
+  (let [trim-email (fn [s] (subs s 1 (dec (count s))))
+        dt (fn [ds] (Date. (* 1000 (Integer/parseInt ds))))
+        [tree parents author committer msg]
+        (with-open [s (exec-stream (str "git cat-file -p " sha))]
+          (let [lines (line-seq s)
+                slines (mapv #(string/split % #"\s") lines)
+                tree (-> slines (nth 0) (nth 1))
+                [plines xs] (split-with #(= (nth % 0) "parent") (rest slines))]
+            [tree
+             (seq (map second plines))
+             (vec (reverse (first xs)))
+             (vec (reverse (second xs)))
+             (->> lines
+                  (drop-while #(not= % ""))
+                  rest
+                  (interpose "\n")
+                  (apply str))]))]
+    {:sha sha
+     :msg msg
+     :tree tree
+     :parents parents
+     :author (trim-email (author 2))
+     :authored (dt (author 1))
+     :committer (trim-email (committer 2))
+     :committed (dt (committer 1))}))
 
 
 
 (defn commit-tx-data
-  [grepo db repo repo-name {:keys [sha msg tree parents author authored committer committed] :as commit}]
+  [db repo repo-name {:keys [sha msg tree parents author authored committer committed] :as commit}]
   (let [tempid? map? ;;todo - better pred
         sha->id (index->id-fn db :git/sha)
         email->id (index->id-fn db :email/address)
@@ -319,7 +379,7 @@
                                      newpath (conj [:db/add nodeid :node/paths pathid])
                                      (tempid? id) (conj {:db/id id :git/sha sha :git/type type}))
                         data (if (and newpath (= type :tree))
-                               (let [es (git/tree-entries grepo sha)]
+                               (let [es (dir sha)]
                                  (reduce (fn [data child]
                                            (let [[cid cdata] (f (str path "/") child)
                                                  data (into data cdata)]
@@ -359,17 +419,27 @@
                    (conj [:db/add committerid :email/address committer]))]
     tx))
 
+(defn commits
+  "Returns log as [[sha msg] ...], in commit order. commit-name may be nil
+  or any acceptable commit name arg for git log"
+  [commit-name]
+  (let [commits (with-open [s (exec-stream (str "git log --pretty=oneline --date-order --reverse " commit-name))]
+                  (mapv
+                   #(vector (subs % 0 40)
+                            (subs % 41 (count %)))
+                   (line-seq s)))]
+    commits))
+
 (defn unimported-commits
-  [grepo db commit-name]
+  [db commit-name]
   (let [imported (into {}
                        (d/q '[:find ?sha ?e
                               :where
                               [?tx :tx/op :import]
                               [?tx :tx/commit ?e]
                               [?e :git/sha ?sha]]
                             db))]
-    (map (fn [[sha _]] (git/commit-info grepo sha))
-         (remove (fn [[sha _]] (imported sha)) (git/commit-shas grepo commit-name)))))
+    (pmap commit (remove (fn [[sha _]] (imported sha)) (commits commit-name)))))
 
 
 (defn ensure-db [db-uri]
@@ -379,11 +449,10 @@
     conn))
 
 (defn import-git
-  [grepo conn repo-uri repo-name commits]
+  [conn repo-uri repo-name commits]
   ;;todo - add already existing commits to new repo if it includes them
   (println "Importing repo:" repo-uri "as:" repo-name)
-  (let [start (System/nanoTime)
-        db (d/db conn)
+  (let [db (d/db conn)
         repo
         (or (ffirst (d/q '[:find ?e :in $ ?uri :where [?e :repo/uri ?uri]] db repo-uri))
             (let [temp (d/tempid :db.part/user)
@@ -394,17 +463,15 @@
     (doseq [commit commits]
       (let [db (d/db conn)]
         (println "Importing commit:" (:sha commit))
-        (d/transact conn (commit-tx-data grepo db repo repo-name commit))))
+        (d/transact conn (commit-tx-data db repo repo-name commit))))
     (d/request-index conn)
-    (println "Import complete!"
-             (format "(%.1fs)" (/ (- (System/nanoTime) start) 1e9)))))
+    (println "Import complete!")))
 
 (def analyzers [(datomic.codeq.analyzers.clj/impl)])
 
 (defn run-analyzers
-  [grepo conn]
+  [conn]
   (println "Analyzing...")
-  (let [start (System/nanoTime)]
   (doseq [a analyzers]
     (let [aname (az/keyname a)
           exts (az/extensions a)
@@ -443,7 +510,8 @@
           ;;analyze them
           (println "analyzing file:" f " - sha: " (:git/sha (d/entity db f)))
           (let [db (d/db conn)
-                src (git/blob-text grepo (:git/sha (d/entity db f)))
+                src (with-open [s (exec-stream (str "git cat-file -p " (:git/sha (d/entity db f))))]
+                      (slurp s))
                 adata (try
                         (az/analyze a db f src)
                         (catch Exception ex
@@ -455,18 +523,16 @@
                                      :tx/file f
                                      :tx/analyzer aname
                                      :tx/analyzerRev arev})))))))
-    (println "Analysis complete!"
-             (format "(%.1fs)" (/ (- (System/nanoTime) start) 1e9)))))
+  (println "Analysis complete!"))
 
 (defn main [& [db-uri commit]]
   (if db-uri
-    (with-open [grepo (git/open-repo)]
       (let [conn (ensure-db db-uri)
-            [repo-uri repo-name] (git/repo-uri grepo)]
-        (import-git grepo conn repo-uri repo-name
-                    (unimported-commits grepo (d/db conn) commit))
-        (run-analyzers grepo conn)))
-    (println "Usage: datomic.codeq.core db-uri [commit-name]")))
+            [repo-uri repo-name] (get-repo-uri)]
+        ;;(prn repo-uri)
+        (import-git conn repo-uri repo-name (unimported-commits (d/db conn) commit))
+        (run-analyzers conn))
+      (println "Usage: datomic.codeq.core db-uri [commit-name]")))
 
 (defn -main
   [& args]
@@ -477,7 +543,7 @@
 
 (comment
 (def uri "datomic:mem://git")
-;;(def uri "datomic:dev://localhost:4334/git")
+;;(def uri "datomic:free://localhost:4334/git")
 (datomic.codeq.core/main uri "c3bd979cfe65da35253b25cb62aad4271430405c")
 (datomic.codeq.core/main uri  "20f8db11804afc8c5a1752257d5fdfcc2d131d08")
 (datomic.codeq.core/main uri)