From eca2ea4b58ba33d7c996510391340e89d49257ec Mon Sep 17 00:00:00 2001 From: Devin Walters <10421+devn@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:12:56 -0500 Subject: [PATCH 1/7] Revert "time the import and analysis" This reverts commit e9cf28625148bec72eac7a4844c1eaa3688c35e1. --- src/datomic/codeq/core.clj | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/datomic/codeq/core.clj b/src/datomic/codeq/core.clj index 1615f08..fcd6731 100644 --- a/src/datomic/codeq/core.clj +++ b/src/datomic/codeq/core.clj @@ -382,8 +382,7 @@ [grepo conn repo-uri repo-name commits] ;;todo - add already existing commits to new repo if it includes them (println "Importing repo:" repo-uri "as:" repo-name) - (let [start (System/nanoTime) - db (d/db conn) + (let [db (d/db conn) repo (or (ffirst (d/q '[:find ?e :in $ ?uri :where [?e :repo/uri ?uri]] db repo-uri)) (let [temp (d/tempid :db.part/user) @@ -396,15 +395,13 @@ (println "Importing commit:" (:sha commit)) (d/transact conn (commit-tx-data grepo db repo repo-name commit)))) (d/request-index conn) - (println "Import complete!" - (format "(%.1fs)" (/ (- (System/nanoTime) start) 1e9))))) + (println "Import complete!"))) (def analyzers [(datomic.codeq.analyzers.clj/impl)]) (defn run-analyzers [grepo conn] (println "Analyzing...") - (let [start (System/nanoTime)] (doseq [a analyzers] (let [aname (az/keyname a) exts (az/extensions a) @@ -455,8 +452,7 @@ :tx/file f :tx/analyzer aname :tx/analyzerRev arev}))))))) - (println "Analysis complete!" - (format "(%.1fs)" (/ (- (System/nanoTime) start) 1e9))))) + (println "Analysis complete!")) (defn main [& [db-uri commit]] (if db-uri From b002af6e00520c743a348305126b80a626f816e6 Mon Sep 17 00:00:00 2001 From: Devin Walters <10421+devn@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:12:56 -0500 Subject: [PATCH 2/7] Revert "import via jgit, drop git subprocess" This reverts commit db6ec710ef3b713aeaa1c3f86a088a4573c557fb. --- src/datomic/codeq/core.clj | 102 +++++++++++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 16 deletions(-) diff --git a/src/datomic/codeq/core.clj b/src/datomic/codeq/core.clj index fcd6731..9300e0f 100644 --- a/src/datomic/codeq/core.clj +++ b/src/datomic/codeq/core.clj @@ -8,12 +8,13 @@ (ns datomic.codeq.core (:require [datomic.api :as d] + [clojure.java.io :as io] [clojure.set] [clojure.string :as string] [datomic.codeq.util :refer [index->id-fn tempid?]] - [datomic.codeq.git :as git] [datomic.codeq.analyzer :as az] [datomic.codeq.analyzers.clj]) + (:import java.util.Date) (:gen-class)) (set! *warn-on-reflection* true) @@ -249,6 +250,13 @@ :db.install/_attribute :db.part/db} ]) +(defn ^java.io.Reader exec-stream + [^String cmd] + (-> (Runtime/getRuntime) + (.exec cmd) + .getInputStream + io/reader)) + (defn ensure-schema [conn] (or (-> conn d/db (d/entid :tx/commit)) @(d/transact conn schema))) @@ -285,11 +293,63 @@ ;; Push URL: https://github.com/Datomic/codeq.git ;; HEAD branch: (not queried) +(defn get-repo-uri + "returns [uri name]" + [] + (with-open [s (exec-stream (str "git remote show -n origin"))] + (let [es (line-seq s) + ^String line (second es) + uri (subs line (inc (.lastIndexOf line " "))) + noff (.lastIndexOf uri "/") + noff (if (not (pos? noff)) (.lastIndexOf uri ":") noff) + name (subs uri (inc noff)) + _ (assert (pos? (count name)) "Can't find remote origin") + name (if (.endsWith name ".git") (subs name 0 (.indexOf name ".")) name)] + [uri name]))) + +(defn dir + "Returns [[sha :type filename] ...]" + [tree] + (with-open [s (exec-stream (str "git cat-file -p " tree))] + (let [es (line-seq s)] + (mapv #(let [ss (string/split ^String % #"\s")] + [(nth ss 2) + (keyword (nth ss 1)) + (subs % (inc (.indexOf ^String % "\t")) (count %))]) + es)))) + +(defn commit + [[sha _]] + (let [trim-email (fn [s] (subs s 1 (dec (count s)))) + dt (fn [ds] (Date. (* 1000 (Integer/parseInt ds)))) + [tree parents author committer msg] + (with-open [s (exec-stream (str "git cat-file -p " sha))] + (let [lines (line-seq s) + slines (mapv #(string/split % #"\s") lines) + tree (-> slines (nth 0) (nth 1)) + [plines xs] (split-with #(= (nth % 0) "parent") (rest slines))] + [tree + (seq (map second plines)) + (vec (reverse (first xs))) + (vec (reverse (second xs))) + (->> lines + (drop-while #(not= % "")) + rest + (interpose "\n") + (apply str))]))] + {:sha sha + :msg msg + :tree tree + :parents parents + :author (trim-email (author 2)) + :authored (dt (author 1)) + :committer (trim-email (committer 2)) + :committed (dt (committer 1))})) (defn commit-tx-data - [grepo db repo repo-name {:keys [sha msg tree parents author authored committer committed] :as commit}] + [db repo repo-name {:keys [sha msg tree parents author authored committer committed] :as commit}] (let [tempid? map? ;;todo - better pred sha->id (index->id-fn db :git/sha) email->id (index->id-fn db :email/address) @@ -319,7 +379,7 @@ newpath (conj [:db/add nodeid :node/paths pathid]) (tempid? id) (conj {:db/id id :git/sha sha :git/type type})) data (if (and newpath (= type :tree)) - (let [es (git/tree-entries grepo sha)] + (let [es (dir sha)] (reduce (fn [data child] (let [[cid cdata] (f (str path "/") child) data (into data cdata)] @@ -359,8 +419,19 @@ (conj [:db/add committerid :email/address committer]))] tx)) +(defn commits + "Returns log as [[sha msg] ...], in commit order. commit-name may be nil + or any acceptable commit name arg for git log" + [commit-name] + (let [commits (with-open [s (exec-stream (str "git log --pretty=oneline --date-order --reverse " commit-name))] + (mapv + #(vector (subs % 0 40) + (subs % 41 (count %))) + (line-seq s)))] + commits)) + (defn unimported-commits - [grepo db commit-name] + [db commit-name] (let [imported (into {} (d/q '[:find ?sha ?e :where @@ -368,8 +439,7 @@ [?tx :tx/commit ?e] [?e :git/sha ?sha]] db))] - (map (fn [[sha _]] (git/commit-info grepo sha)) - (remove (fn [[sha _]] (imported sha)) (git/commit-shas grepo commit-name))))) + (pmap commit (remove (fn [[sha _]] (imported sha)) (commits commit-name))))) (defn ensure-db [db-uri] @@ -379,7 +449,7 @@ conn)) (defn import-git - [grepo conn repo-uri repo-name commits] + [conn repo-uri repo-name commits] ;;todo - add already existing commits to new repo if it includes them (println "Importing repo:" repo-uri "as:" repo-name) (let [db (d/db conn) @@ -393,14 +463,14 @@ (doseq [commit commits] (let [db (d/db conn)] (println "Importing commit:" (:sha commit)) - (d/transact conn (commit-tx-data grepo db repo repo-name commit)))) + (d/transact conn (commit-tx-data db repo repo-name commit)))) (d/request-index conn) (println "Import complete!"))) (def analyzers [(datomic.codeq.analyzers.clj/impl)]) (defn run-analyzers - [grepo conn] + [conn] (println "Analyzing...") (doseq [a analyzers] (let [aname (az/keyname a) @@ -440,7 +510,8 @@ ;;analyze them (println "analyzing file:" f " - sha: " (:git/sha (d/entity db f))) (let [db (d/db conn) - src (git/blob-text grepo (:git/sha (d/entity db f))) + src (with-open [s (exec-stream (str "git cat-file -p " (:git/sha (d/entity db f))))] + (slurp s)) adata (try (az/analyze a db f src) (catch Exception ex @@ -456,13 +527,12 @@ (defn main [& [db-uri commit]] (if db-uri - (with-open [grepo (git/open-repo)] (let [conn (ensure-db db-uri) - [repo-uri repo-name] (git/repo-uri grepo)] - (import-git grepo conn repo-uri repo-name - (unimported-commits grepo (d/db conn) commit)) - (run-analyzers grepo conn))) - (println "Usage: datomic.codeq.core db-uri [commit-name]"))) + [repo-uri repo-name] (get-repo-uri)] + ;;(prn repo-uri) + (import-git conn repo-uri repo-name (unimported-commits (d/db conn) commit)) + (run-analyzers conn)) + (println "Usage: datomic.codeq.core db-uri [commit-name]"))) (defn -main [& args] From 72c26edc0ee81f73f258588e6ba95f24cde50709 Mon Sep 17 00:00:00 2001 From: Devin Walters <10421+devn@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:12:56 -0500 Subject: [PATCH 3/7] Revert "read repo uri via jgit" This reverts commit 83d531ad2f476c0c1835944b4466b5a1311f2b61. --- src/datomic/codeq/git.clj | 14 -------------- test/datomic/codeq/git_test.clj | 9 --------- 2 files changed, 23 deletions(-) diff --git a/src/datomic/codeq/git.clj b/src/datomic/codeq/git.clj index 60aff56..6a30a08 100644 --- a/src/datomic/codeq/git.clj +++ b/src/datomic/codeq/git.clj @@ -74,20 +74,6 @@ acc)) (finally (.close tw))))) -(defn repo-uri - "[uri name] from remote.origin.url." - [^Repository repo] - (let [^String uri (.getString (.getConfig repo) "remote" "origin" "url")] - (assert uri "Can't find remote origin") - (let [noff (.lastIndexOf uri "/") - noff (if (pos? noff) noff (.lastIndexOf uri ":")) - name (subs uri (inc noff)) - _ (assert (pos? (count name)) "Can't find remote origin") - name (if (.endsWith name ".git") - (subs name 0 (.indexOf name ".")) - name)] - [uri name]))) - (defn commit-info "Map of commit fields for the 40-char hex sha." [^Repository repo ^String sha] diff --git a/test/datomic/codeq/git_test.clj b/test/datomic/codeq/git_test.clj index 4a09ad8..b100397 100644 --- a/test/datomic/codeq/git_test.clj +++ b/test/datomic/codeq/git_test.clj @@ -75,15 +75,6 @@ shas (mapv first (git/commit-shas repo nil))] (is (= [(.name c1) (.name c2)] shas))))))) -(deftest repo-uri-reads-origin - (testing "repo-uri returns [uri name] from remote.origin.url" - (with-temp-repo - (fn [dir git ^Repository repo] - (let [cfg (.getConfig repo)] - (.setString cfg "remote" "origin" "url" "git@github.com:devn/codeq.git") - (.save cfg)) - (is (= ["git@github.com:devn/codeq.git" "codeq"] (git/repo-uri repo))))))) - (deftest tree-entries-lists-direct-children (testing "tree-entries returns [sha type filename] for top-level entries" (with-temp-repo From f999b48ce4ccc4399a0adb38a72c53e167576b31 Mon Sep 17 00:00:00 2001 From: Devin Walters <10421+devn@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:12:56 -0500 Subject: [PATCH 4/7] Revert "read trees via jgit" This reverts commit e11ffcf6d0a6d4db87e24ee16dd54cb389ad0f69. --- src/datomic/codeq/git.clj | 21 --------------------- test/datomic/codeq/git_test.clj | 18 ------------------ 2 files changed, 39 deletions(-) diff --git a/src/datomic/codeq/git.clj b/src/datomic/codeq/git.clj index 6a30a08..4b1ba03 100644 --- a/src/datomic/codeq/git.clj +++ b/src/datomic/codeq/git.clj @@ -53,27 +53,6 @@ (iterator-seq (.iterator walk)))) (finally (.close walk))))) -(defn- mode->type [^FileMode m] - (cond - (= m FileMode/TREE) :tree - (= m FileMode/GITLINK) :commit - :else :blob)) - -(defn tree-entries - "[[sha :type filename] ...] for the direct children of tree-sha, in git tree order." - [^Repository repo ^String tree-sha] - (let [tw (doto (TreeWalk. repo) - (.addTree (ObjectId/fromString tree-sha)) - (.setRecursive false))] - (try - (loop [acc []] - (if (.next tw) - (recur (conj acc [(.name (.getObjectId tw 0)) - (mode->type (.getFileMode tw 0)) - (.getNameString tw)])) - acc)) - (finally (.close tw))))) - (defn commit-info "Map of commit fields for the 40-char hex sha." [^Repository repo ^String sha] diff --git a/test/datomic/codeq/git_test.clj b/test/datomic/codeq/git_test.clj index b100397..c50289a 100644 --- a/test/datomic/codeq/git_test.clj +++ b/test/datomic/codeq/git_test.clj @@ -74,21 +74,3 @@ c2 (commit-file! git dir "a.txt" "one\ntwo\n" "second") shas (mapv first (git/commit-shas repo nil))] (is (= [(.name c1) (.name c2)] shas))))))) - -(deftest tree-entries-lists-direct-children - (testing "tree-entries returns [sha type filename] for top-level entries" - (with-temp-repo - (fn [dir git ^Repository repo] - (.mkdirs (clojure.java.io/file dir "sub")) - (spit (clojure.java.io/file dir "sub" "nested.txt") "n\n") - (let [rc (commit-file! git dir "top.txt" "t\n" "mixed tree") - ;; also stage the nested dir - _ (.. git add (addFilepattern "sub") (call)) - rc2 (.. git commit (setMessage "with sub") - (setAuthor "Ada Lovelace" "ada@example.com") - (setCommitter "Ada Lovelace" "ada@example.com") (call)) - entries (git/tree-entries repo (.name (.getTree rc2))) - by-name (into {} (map (fn [[sha type fname]] [fname [type]]) entries))] - (is (= [:blob] (by-name "top.txt"))) - (is (= [:tree] (by-name "sub"))) - (is (every? #(= 40 (count (first %))) entries))))))) From c1213bfd5c3bdf2c17b1c3802da048f76c0c4d80 Mon Sep 17 00:00:00 2001 From: Devin Walters <10421+devn@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:12:56 -0500 Subject: [PATCH 5/7] Revert "read commits and ordering via jgit" This reverts commit ce75d5818874b34a4ea4280e10b81f2d46456365. --- src/datomic/codeq/git.clj | 36 --------------------------------- test/datomic/codeq/git_test.clj | 25 ----------------------- 2 files changed, 61 deletions(-) diff --git a/src/datomic/codeq/git.clj b/src/datomic/codeq/git.clj index 4b1ba03..3b47370 100644 --- a/src/datomic/codeq/git.clj +++ b/src/datomic/codeq/git.clj @@ -35,39 +35,3 @@ (let [loader (.open reader (ObjectId/fromString sha))] (String. (.getBytes loader) StandardCharsets/UTF_8)) (finally (.close reader))))) - -(defn commit-shas - "[[sha short-message] ...] in import order (parents before children). - rev nil => HEAD." - [^Repository repo rev] - (let [walk (RevWalk. repo)] - (try - (let [start (.resolve repo (or rev Constants/HEAD))] - (when (nil? start) - (throw (ex-info (str "Could not resolve rev: " (or rev "HEAD")) - {:rev rev}))) - (.markStart walk (.parseCommit walk start)) - (.sort walk RevSort/TOPO) - (.sort walk RevSort/REVERSE true) - (mapv (fn [^RevCommit c] [(.name c) (.getShortMessage c)]) - (iterator-seq (.iterator walk)))) - (finally (.close walk))))) - -(defn commit-info - "Map of commit fields for the 40-char hex sha." - [^Repository repo ^String sha] - (let [walk (RevWalk. repo)] - (try - (let [^RevCommit c (.parseCommit walk (ObjectId/fromString sha)) - ^PersonIdent author (.getAuthorIdent c) - ^PersonIdent committer (.getCommitterIdent c) - parents (mapv #(.name ^RevCommit %) (.getParents c))] - {:sha sha - :msg (string/trimr (.getFullMessage c)) - :tree (.name (.getTree c)) - :parents (seq parents) - :author (.getEmailAddress author) - :authored (.getWhen author) - :committer (.getEmailAddress committer) - :committed (.getWhen committer)}) - (finally (.close walk))))) diff --git a/test/datomic/codeq/git_test.clj b/test/datomic/codeq/git_test.clj index c50289a..9610f61 100644 --- a/test/datomic/codeq/git_test.clj +++ b/test/datomic/codeq/git_test.clj @@ -49,28 +49,3 @@ blob-sha (.name (.getObjectId tw 0))] (.close tw) (is (= "hello world\n" (git/blob-text repo blob-sha)))))))) - -(deftest commit-info-parses-fields - (testing "commit-info returns author email, message, tree, and parents" - (with-temp-repo - (fn [dir git ^Repository repo] - (let [c1 (commit-file! git dir "a.txt" "one\n" "first") - c2 (commit-file! git dir "b.txt" "two\n" "second") - info1 (git/commit-info repo (.name c1)) - info2 (git/commit-info repo (.name c2))] - (is (= (.name c1) (:sha info1))) - (is (= "first" (:msg info1))) - (is (= "ada@example.com" (:author info1))) - (is (= "ada@example.com" (:committer info1))) - (is (instance? java.util.Date (:authored info1))) - (is (nil? (:parents info1))) - (is (= [(.name c1)] (vec (:parents info2))))))))) - -(deftest commit-shas-orders-parents-first - (testing "commit-shas returns commits parents-before-children" - (with-temp-repo - (fn [dir git ^Repository repo] - (let [c1 (commit-file! git dir "a.txt" "one\n" "first") - c2 (commit-file! git dir "a.txt" "one\ntwo\n" "second") - shas (mapv first (git/commit-shas repo nil))] - (is (= [(.name c1) (.name c2)] shas))))))) From 17c3d78172dbae4474bcfdad370679c4a936a91e Mon Sep 17 00:00:00 2001 From: Devin Walters <10421+devn@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:12:56 -0500 Subject: [PATCH 6/7] Revert "add jgit git layer with blob reading" This reverts commit bcddc1c4e88460e9ad0ee79431333786b70a83a8. --- deps.edn | 3 +- src/datomic/codeq/git.clj | 37 ------------------------ test/datomic/codeq/git_test.clj | 51 --------------------------------- 3 files changed, 1 insertion(+), 90 deletions(-) delete mode 100644 src/datomic/codeq/git.clj delete mode 100644 test/datomic/codeq/git_test.clj diff --git a/deps.edn b/deps.edn index 0e47992..698fb41 100644 --- a/deps.edn +++ b/deps.edn @@ -1,8 +1,7 @@ {:paths ["src"] :deps {org.clojure/clojure {:mvn/version "1.12.1"} com.datomic/peer {:mvn/version "1.0.7469" - :exclusions [org.clojure/clojure]} - org.eclipse.jgit/org.eclipse.jgit {:mvn/version "6.10.0.202406032230-r"}} + :exclusions [org.clojure/clojure]}} :aliases {:run {:main-opts ["-m" "datomic.codeq.core"]} :test {:extra-paths ["test"] diff --git a/src/datomic/codeq/git.clj b/src/datomic/codeq/git.clj deleted file mode 100644 index 3b47370..0000000 --- a/src/datomic/codeq/git.clj +++ /dev/null @@ -1,37 +0,0 @@ -;; Copyright (c) Metadata Partners, LLC and Contributors. All rights reserved. -;; The use and distribution terms for this software are covered by the -;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) -;; which can be found in the file epl-v10.html at the root of this distribution. -;; By using this software in any fashion, you are agreeing to be bound by -;; the terms of this license. -;; You must not remove this notice, or any other, from this software. - -(ns datomic.codeq.git - (:require [clojure.string :as string]) - (:import [java.io File] - [java.nio.charset StandardCharsets] - [org.eclipse.jgit.lib Repository ObjectId FileMode PersonIdent Constants] - [org.eclipse.jgit.storage.file FileRepositoryBuilder] - [org.eclipse.jgit.revwalk RevWalk RevCommit RevSort] - [org.eclipse.jgit.treewalk TreeWalk])) - -(set! *warn-on-reflection* true) - -(defn ^Repository open-repo - "Open the git repository discovered from dir (defaults to cwd)." - ([] (open-repo (System/getProperty "user.dir"))) - ([dir] - (-> (FileRepositoryBuilder.) - (.findGitDir (File. ^String (str dir))) - (.setMustExist true) - (.readEnvironment) - (.build)))) - -(defn ^String blob-text - "UTF-8 text of the blob named by the 40-char hex sha." - [^Repository repo ^String sha] - (let [reader (.newObjectReader repo)] - (try - (let [loader (.open reader (ObjectId/fromString sha))] - (String. (.getBytes loader) StandardCharsets/UTF_8)) - (finally (.close reader))))) diff --git a/test/datomic/codeq/git_test.clj b/test/datomic/codeq/git_test.clj deleted file mode 100644 index 9610f61..0000000 --- a/test/datomic/codeq/git_test.clj +++ /dev/null @@ -1,51 +0,0 @@ -(ns datomic.codeq.git-test - (:require [clojure.test :refer [deftest testing is]] - [clojure.java.io :as io] - [datomic.codeq.git :as git]) - (:import [org.eclipse.jgit.api Git] - [org.eclipse.jgit.lib Repository] - [java.io File])) - -(defn- temp-dir ^File [] - (let [d (File/createTempFile "codeq-git-test" "")] - (.delete d) (.mkdirs d) d)) - -(defn- delete-recursively [^File f] - (when (.isDirectory f) - (doseq [c (.listFiles f)] (delete-recursively c))) - (.delete f)) - -(defn with-temp-repo - "Calls (f dir git repo) with a fresh initialized repo; cleans up after." - [f] - (let [dir (temp-dir) - git (.. (Git/init) (setDirectory dir) (call)) - repo (.getRepository git)] - (try (f dir git repo) - (finally (.close git) (delete-recursively dir))))) - -(defn commit-file! - "Writes fname=content, stages, commits with a deterministic author. Returns RevCommit." - [^Git git ^File dir ^String fname ^String content ^String msg] - (spit (io/file dir fname) content) - (.. git add (addFilepattern fname) (call)) - (.. git commit - (setMessage msg) - (setAuthor "Ada Lovelace" "ada@example.com") - (setCommitter "Ada Lovelace" "ada@example.com") - (call))) - -(deftest blob-text-round-trips - (testing "blob-text returns the file content for a committed blob" - (with-temp-repo - (fn [dir git ^Repository repo] - (let [rc (commit-file! git dir "hello.txt" "hello world\n" "add hello") - tree-sha (.name (.getTree rc)) - ;; find the blob sha via JGit TreeWalk in the test - tw (doto (org.eclipse.jgit.treewalk.TreeWalk. repo) - (.addTree (org.eclipse.jgit.lib.ObjectId/fromString tree-sha)) - (.setRecursive true)) - _ (.next tw) - blob-sha (.name (.getObjectId tw 0))] - (.close tw) - (is (= "hello world\n" (git/blob-text repo blob-sha)))))))) From 54200b395de0aeed4f09c964fe40b23193d2aa9c Mon Sep 17 00:00:00 2001 From: Devin Walters <10421+devn@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:12:56 -0500 Subject: [PATCH 7/7] Revert "Modernize with datomic free, deps.edn" This reverts commit 39593d8da98a509fc0cd5338167fdd68cae9abf6. --- .gitignore | 8 +++++++- README.md | 22 +++++++++++----------- build.clj | 21 --------------------- deps.edn | 12 ------------ project.clj | 11 +++++++++++ src/datomic/codeq/analyzer.clj | 7 +++---- src/datomic/codeq/core.clj | 2 +- test/datomic/codeq/analyzer_test.clj | 11 ----------- 8 files changed, 33 insertions(+), 61 deletions(-) delete mode 100644 build.clj delete mode 100644 deps.edn create mode 100644 project.clj delete mode 100644 test/datomic/codeq/analyzer_test.clj diff --git a/.gitignore b/.gitignore index 215f475..cc989ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ /target +/lib /classes -.cpcache/ +/checkouts +/tmp pom.xml *.jar *.class +.lein-deps-sum +.lein-failures +.lein-plugins +*.tar diff --git a/README.md b/README.md index 4153759..93b5100 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,17 @@ ## Usage -Clone the **codeq** repo. Then, with the [Clojure CLI](https://clojure.org/guides/install_clojure) installed: +Clone the **codeq** repo. Then (in it) run: -Build a standalone jar: + lein uberjar - clojure -T:build uber +Get [Datomic Free](http://www.datomic.com/get-datomic.html) -`codeq` uses [Datomic Pro](https://www.datomic.com/) (free, no license required) via the Peer API. For a quick run with no transactor, use the in-memory `datomic:mem://` storage: +Unzip it, then start the Datomic Free transactor. Follow the instructions for [running the transactor with the free storage protocol](http://docs.datomic.com/getting-started.html) cd theGitRepoYouWantToImport - java -jar whereverYouPutCodeq/target/codeq.jar datomic:mem://git -For development without building a jar: - - clojure -M:run datomic:mem://git - -For persistent storage, start a Datomic Pro transactor and pass its URI (e.g. `datomic:dev://localhost:4334/git`) instead of `datomic:mem://`. + java -server -Xmx1g -jar whereverYouPutCodeq/target/codeq-0.1.0-SNAPSHOT-standalone.jar datomic:free://localhost:4334/git This will create a db called `git` (you can call it whatever you like) and import the commits from the local view of the repo. You should see output like: @@ -43,7 +38,12 @@ The import is not too peppy, since it shells to `git` relentlessly, but it impor You can import more than one repo into the same db. You can re-import later after some more commits and they will be incrementally added. -You can then (or during) connect to the same db URI with a peer. Note that the in-memory `datomic:mem://` db only lives inside the importing process — to browse a db after import, run against a persistent storage URI backed by a Datomic Pro transactor (e.g. `datomic:dev://localhost:4334/git`). +You can then (or during) connect to the same db URI with a peer. Or, just start the [Datomic REST service](http://docs.datomic.com/rest.html) and poke around: + + cd whereverYouPutDatomicFree + bin/rest -p 8080 free datomic:free://localhost:4334/ + +Browse to [localhost:8080/data/](http://localhost:8080/data/). You should see the `free` storage and the `git` db within it. The [schema diagram](https://github.com/downloads/Datomic/codeq/codeq.pdf) will help you get oriented. diff --git a/build.clj b/build.clj deleted file mode 100644 index 01b2f9a..0000000 --- a/build.clj +++ /dev/null @@ -1,21 +0,0 @@ -(ns build - (:require [clojure.tools.build.api :as b])) - -(def class-dir "target/classes") -(def uber-file "target/codeq.jar") -(def basis (delay (b/create-basis {:project "deps.edn"}))) - -(defn clean [_] - (b/delete {:path "target"})) - -(defn uber [_] - (clean nil) - (b/copy-dir {:src-dirs ["src"] - :target-dir class-dir}) - (b/compile-clj {:basis @basis - :ns-compile '[datomic.codeq.core] - :class-dir class-dir}) - (b/uber {:class-dir class-dir - :uber-file uber-file - :basis @basis - :main 'datomic.codeq.core})) diff --git a/deps.edn b/deps.edn deleted file mode 100644 index 698fb41..0000000 --- a/deps.edn +++ /dev/null @@ -1,12 +0,0 @@ -{:paths ["src"] - :deps {org.clojure/clojure {:mvn/version "1.12.1"} - com.datomic/peer {:mvn/version "1.0.7469" - :exclusions [org.clojure/clojure]}} - :aliases - {:run {:main-opts ["-m" "datomic.codeq.core"]} - :test {:extra-paths ["test"] - :extra-deps {io.github.cognitect-labs/test-runner - {:git/tag "v0.5.1" :git/sha "dfb30dd"}} - :main-opts ["-m" "cognitect.test-runner"]} - :build {:deps {io.github.clojure/tools.build {:mvn/version "0.10.5"}} - :ns-default build}}} diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..1ce950d --- /dev/null +++ b/project.clj @@ -0,0 +1,11 @@ +(defproject datomic/codeq "0.1.0-SNAPSHOT" + :description "codeq does a code-aware import of your git repo into a Datomic db" + :url "http://datomic.com" + :license {:name "Eclipse Public License" + :url "http://www.eclipse.org/legal/epl-v10.html"} + :main datomic.codeq.core + :plugins [[lein-tar "1.1.0"]] + :dependencies [[com.datomic/datomic-free "0.9.4699"] + [commons-codec "1.7"] + [org.clojure/clojure "1.5.1"]] + :source-paths ["src" "examples/src"]) diff --git a/src/datomic/codeq/analyzer.clj b/src/datomic/codeq/analyzer.clj index a746aca..4d662eb 100644 --- a/src/datomic/codeq/analyzer.clj +++ b/src/datomic/codeq/analyzer.clj @@ -7,7 +7,8 @@ ;; You must not remove this notice, or any other, from this software. (ns datomic.codeq.analyzer - (:import [java.io StringReader])) + (:import [java.io StringReader] + [org.apache.commons.codec.digest DigestUtils])) (set! *warn-on-reflection* true) @@ -21,9 +22,7 @@ (defn sha "Returns the hex string of the sha1 of s" [^String s] - (let [bytes (.digest (java.security.MessageDigest/getInstance "SHA-1") - (.getBytes s "UTF-8"))] - (apply str (map #(format "%02x" %) bytes)))) + (org.apache.commons.codec.digest.DigestUtils/shaHex s)) (defn ws-minify "Consecutive ws becomes a single space, then trim" diff --git a/src/datomic/codeq/core.clj b/src/datomic/codeq/core.clj index 9300e0f..ef83b45 100644 --- a/src/datomic/codeq/core.clj +++ b/src/datomic/codeq/core.clj @@ -543,7 +543,7 @@ (comment (def uri "datomic:mem://git") -;;(def uri "datomic:dev://localhost:4334/git") +;;(def uri "datomic:free://localhost:4334/git") (datomic.codeq.core/main uri "c3bd979cfe65da35253b25cb62aad4271430405c") (datomic.codeq.core/main uri "20f8db11804afc8c5a1752257d5fdfcc2d131d08") (datomic.codeq.core/main uri) diff --git a/test/datomic/codeq/analyzer_test.clj b/test/datomic/codeq/analyzer_test.clj deleted file mode 100644 index 21bae11..0000000 --- a/test/datomic/codeq/analyzer_test.clj +++ /dev/null @@ -1,11 +0,0 @@ -(ns datomic.codeq.analyzer-test - (:require [clojure.test :refer [deftest testing is]] - [datomic.codeq.analyzer :as az])) - -(deftest sha-matches-known-sha1 - (testing "sha returns lowercase hex sha1 matching commons-codec shaHex output" - ;; Known SHA-1 values (lowercase hex), verifiable via `printf '%s' "..." | shasum` - (is (= "da39a3ee5e6b4b0d3255bfef95601890afd80709" (az/sha ""))) - (is (= "a9993e364706816aba3e25717850c26c9cd0d89d" (az/sha "abc"))) - (is (= "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33" (az/sha "foo"))) - (is (= 40 (count (az/sha "anything"))))))