From 39593d8da98a509fc0cd5338167fdd68cae9abf6 Mon Sep 17 00:00:00 2001 From: Devin Walters Date: Wed, 17 Jun 2026 18:40:54 -0500 Subject: [PATCH 1/8] Modernize with datomic free, deps.edn --- .gitignore | 8 +------- README.md | 22 +++++++++++----------- build.clj | 21 +++++++++++++++++++++ deps.edn | 12 ++++++++++++ project.clj | 11 ----------- src/datomic/codeq/analyzer.clj | 7 ++++--- src/datomic/codeq/core.clj | 2 +- test/datomic/codeq/analyzer_test.clj | 11 +++++++++++ 8 files changed, 61 insertions(+), 33 deletions(-) create mode 100644 build.clj create mode 100644 deps.edn delete mode 100644 project.clj create mode 100644 test/datomic/codeq/analyzer_test.clj diff --git a/.gitignore b/.gitignore index cc989ce..215f475 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,6 @@ /target -/lib /classes -/checkouts -/tmp +.cpcache/ pom.xml *.jar *.class -.lein-deps-sum -.lein-failures -.lein-plugins -*.tar diff --git a/README.md b/README.md index 93b5100..4153759 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,22 @@ ## Usage -Clone the **codeq** repo. Then (in it) run: +Clone the **codeq** repo. Then, with the [Clojure CLI](https://clojure.org/guides/install_clojure) installed: - lein uberjar +Build a standalone jar: -Get [Datomic Free](http://www.datomic.com/get-datomic.html) + clojure -T:build uber -Unzip it, then start the Datomic Free transactor. Follow the instructions for [running the transactor with the free storage protocol](http://docs.datomic.com/getting-started.html) +`codeq` uses [Datomic Pro](https://www.datomic.com/) (free, no license required) via the Peer API. For a quick run with no transactor, use the in-memory `datomic:mem://` storage: cd theGitRepoYouWantToImport + java -jar whereverYouPutCodeq/target/codeq.jar datomic:mem://git - java -server -Xmx1g -jar whereverYouPutCodeq/target/codeq-0.1.0-SNAPSHOT-standalone.jar datomic:free://localhost:4334/git +For development without building a jar: + + clojure -M:run datomic:mem://git + +For persistent storage, start a Datomic Pro transactor and pass its URI (e.g. `datomic:dev://localhost:4334/git`) instead of `datomic:mem://`. This will create a db called `git` (you can call it whatever you like) and import the commits from the local view of the repo. You should see output like: @@ -38,12 +43,7 @@ The import is not too peppy, since it shells to `git` relentlessly, but it impor You can import more than one repo into the same db. You can re-import later after some more commits and they will be incrementally added. -You can then (or during) connect to the same db URI with a peer. Or, just start the [Datomic REST service](http://docs.datomic.com/rest.html) and poke around: - - cd whereverYouPutDatomicFree - bin/rest -p 8080 free datomic:free://localhost:4334/ - -Browse to [localhost:8080/data/](http://localhost:8080/data/). You should see the `free` storage and the `git` db within it. +You can then (or during) connect to the same db URI with a peer. Note that the in-memory `datomic:mem://` db only lives inside the importing process — to browse a db after import, run against a persistent storage URI backed by a Datomic Pro transactor (e.g. `datomic:dev://localhost:4334/git`). The [schema diagram](https://github.com/downloads/Datomic/codeq/codeq.pdf) will help you get oriented. diff --git a/build.clj b/build.clj new file mode 100644 index 0000000..01b2f9a --- /dev/null +++ b/build.clj @@ -0,0 +1,21 @@ +(ns build + (:require [clojure.tools.build.api :as b])) + +(def class-dir "target/classes") +(def uber-file "target/codeq.jar") +(def basis (delay (b/create-basis {:project "deps.edn"}))) + +(defn clean [_] + (b/delete {:path "target"})) + +(defn uber [_] + (clean nil) + (b/copy-dir {:src-dirs ["src"] + :target-dir class-dir}) + (b/compile-clj {:basis @basis + :ns-compile '[datomic.codeq.core] + :class-dir class-dir}) + (b/uber {:class-dir class-dir + :uber-file uber-file + :basis @basis + :main 'datomic.codeq.core})) diff --git a/deps.edn b/deps.edn new file mode 100644 index 0000000..698fb41 --- /dev/null +++ b/deps.edn @@ -0,0 +1,12 @@ +{:paths ["src"] + :deps {org.clojure/clojure {:mvn/version "1.12.1"} + com.datomic/peer {:mvn/version "1.0.7469" + :exclusions [org.clojure/clojure]}} + :aliases + {:run {:main-opts ["-m" "datomic.codeq.core"]} + :test {:extra-paths ["test"] + :extra-deps {io.github.cognitect-labs/test-runner + {:git/tag "v0.5.1" :git/sha "dfb30dd"}} + :main-opts ["-m" "cognitect.test-runner"]} + :build {:deps {io.github.clojure/tools.build {:mvn/version "0.10.5"}} + :ns-default build}}} diff --git a/project.clj b/project.clj deleted file mode 100644 index 1ce950d..0000000 --- a/project.clj +++ /dev/null @@ -1,11 +0,0 @@ -(defproject datomic/codeq "0.1.0-SNAPSHOT" - :description "codeq does a code-aware import of your git repo into a Datomic db" - :url "http://datomic.com" - :license {:name "Eclipse Public License" - :url "http://www.eclipse.org/legal/epl-v10.html"} - :main datomic.codeq.core - :plugins [[lein-tar "1.1.0"]] - :dependencies [[com.datomic/datomic-free "0.9.4699"] - [commons-codec "1.7"] - [org.clojure/clojure "1.5.1"]] - :source-paths ["src" "examples/src"]) diff --git a/src/datomic/codeq/analyzer.clj b/src/datomic/codeq/analyzer.clj index 4d662eb..a746aca 100644 --- a/src/datomic/codeq/analyzer.clj +++ b/src/datomic/codeq/analyzer.clj @@ -7,8 +7,7 @@ ;; You must not remove this notice, or any other, from this software. (ns datomic.codeq.analyzer - (:import [java.io StringReader] - [org.apache.commons.codec.digest DigestUtils])) + (:import [java.io StringReader])) (set! *warn-on-reflection* true) @@ -22,7 +21,9 @@ (defn sha "Returns the hex string of the sha1 of s" [^String s] - (org.apache.commons.codec.digest.DigestUtils/shaHex s)) + (let [bytes (.digest (java.security.MessageDigest/getInstance "SHA-1") + (.getBytes s "UTF-8"))] + (apply str (map #(format "%02x" %) bytes)))) (defn ws-minify "Consecutive ws becomes a single space, then trim" diff --git a/src/datomic/codeq/core.clj b/src/datomic/codeq/core.clj index ef83b45..9300e0f 100644 --- a/src/datomic/codeq/core.clj +++ b/src/datomic/codeq/core.clj @@ -543,7 +543,7 @@ (comment (def uri "datomic:mem://git") -;;(def uri "datomic:free://localhost:4334/git") +;;(def uri "datomic:dev://localhost:4334/git") (datomic.codeq.core/main uri "c3bd979cfe65da35253b25cb62aad4271430405c") (datomic.codeq.core/main uri "20f8db11804afc8c5a1752257d5fdfcc2d131d08") (datomic.codeq.core/main uri) diff --git a/test/datomic/codeq/analyzer_test.clj b/test/datomic/codeq/analyzer_test.clj new file mode 100644 index 0000000..21bae11 --- /dev/null +++ b/test/datomic/codeq/analyzer_test.clj @@ -0,0 +1,11 @@ +(ns datomic.codeq.analyzer-test + (:require [clojure.test :refer [deftest testing is]] + [datomic.codeq.analyzer :as az])) + +(deftest sha-matches-known-sha1 + (testing "sha returns lowercase hex sha1 matching commons-codec shaHex output" + ;; Known SHA-1 values (lowercase hex), verifiable via `printf '%s' "..." | shasum` + (is (= "da39a3ee5e6b4b0d3255bfef95601890afd80709" (az/sha ""))) + (is (= "a9993e364706816aba3e25717850c26c9cd0d89d" (az/sha "abc"))) + (is (= "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33" (az/sha "foo"))) + (is (= 40 (count (az/sha "anything")))))) From bcddc1c4e88460e9ad0ee79431333786b70a83a8 Mon Sep 17 00:00:00 2001 From: Devin Walters Date: Wed, 17 Jun 2026 19:19:50 -0500 Subject: [PATCH 2/8] add jgit git layer with blob reading --- deps.edn | 3 +- src/datomic/codeq/git.clj | 37 ++++++++++++++++++++++++ test/datomic/codeq/git_test.clj | 51 +++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 src/datomic/codeq/git.clj create mode 100644 test/datomic/codeq/git_test.clj diff --git a/deps.edn b/deps.edn index 698fb41..0e47992 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,8 @@ {:paths ["src"] :deps {org.clojure/clojure {:mvn/version "1.12.1"} com.datomic/peer {:mvn/version "1.0.7469" - :exclusions [org.clojure/clojure]}} + :exclusions [org.clojure/clojure]} + org.eclipse.jgit/org.eclipse.jgit {:mvn/version "6.10.0.202406032230-r"}} :aliases {:run {:main-opts ["-m" "datomic.codeq.core"]} :test {:extra-paths ["test"] diff --git a/src/datomic/codeq/git.clj b/src/datomic/codeq/git.clj new file mode 100644 index 0000000..3b47370 --- /dev/null +++ b/src/datomic/codeq/git.clj @@ -0,0 +1,37 @@ +;; Copyright (c) Metadata Partners, LLC and Contributors. All rights reserved. +;; The use and distribution terms for this software are covered by the +;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) +;; which can be found in the file epl-v10.html at the root of this distribution. +;; By using this software in any fashion, you are agreeing to be bound by +;; the terms of this license. +;; You must not remove this notice, or any other, from this software. + +(ns datomic.codeq.git + (:require [clojure.string :as string]) + (:import [java.io File] + [java.nio.charset StandardCharsets] + [org.eclipse.jgit.lib Repository ObjectId FileMode PersonIdent Constants] + [org.eclipse.jgit.storage.file FileRepositoryBuilder] + [org.eclipse.jgit.revwalk RevWalk RevCommit RevSort] + [org.eclipse.jgit.treewalk TreeWalk])) + +(set! *warn-on-reflection* true) + +(defn ^Repository open-repo + "Open the git repository discovered from dir (defaults to cwd)." + ([] (open-repo (System/getProperty "user.dir"))) + ([dir] + (-> (FileRepositoryBuilder.) + (.findGitDir (File. ^String (str dir))) + (.setMustExist true) + (.readEnvironment) + (.build)))) + +(defn ^String blob-text + "UTF-8 text of the blob named by the 40-char hex sha." + [^Repository repo ^String sha] + (let [reader (.newObjectReader repo)] + (try + (let [loader (.open reader (ObjectId/fromString sha))] + (String. (.getBytes loader) StandardCharsets/UTF_8)) + (finally (.close reader))))) diff --git a/test/datomic/codeq/git_test.clj b/test/datomic/codeq/git_test.clj new file mode 100644 index 0000000..9610f61 --- /dev/null +++ b/test/datomic/codeq/git_test.clj @@ -0,0 +1,51 @@ +(ns datomic.codeq.git-test + (:require [clojure.test :refer [deftest testing is]] + [clojure.java.io :as io] + [datomic.codeq.git :as git]) + (:import [org.eclipse.jgit.api Git] + [org.eclipse.jgit.lib Repository] + [java.io File])) + +(defn- temp-dir ^File [] + (let [d (File/createTempFile "codeq-git-test" "")] + (.delete d) (.mkdirs d) d)) + +(defn- delete-recursively [^File f] + (when (.isDirectory f) + (doseq [c (.listFiles f)] (delete-recursively c))) + (.delete f)) + +(defn with-temp-repo + "Calls (f dir git repo) with a fresh initialized repo; cleans up after." + [f] + (let [dir (temp-dir) + git (.. (Git/init) (setDirectory dir) (call)) + repo (.getRepository git)] + (try (f dir git repo) + (finally (.close git) (delete-recursively dir))))) + +(defn commit-file! + "Writes fname=content, stages, commits with a deterministic author. Returns RevCommit." + [^Git git ^File dir ^String fname ^String content ^String msg] + (spit (io/file dir fname) content) + (.. git add (addFilepattern fname) (call)) + (.. git commit + (setMessage msg) + (setAuthor "Ada Lovelace" "ada@example.com") + (setCommitter "Ada Lovelace" "ada@example.com") + (call))) + +(deftest blob-text-round-trips + (testing "blob-text returns the file content for a committed blob" + (with-temp-repo + (fn [dir git ^Repository repo] + (let [rc (commit-file! git dir "hello.txt" "hello world\n" "add hello") + tree-sha (.name (.getTree rc)) + ;; find the blob sha via JGit TreeWalk in the test + tw (doto (org.eclipse.jgit.treewalk.TreeWalk. repo) + (.addTree (org.eclipse.jgit.lib.ObjectId/fromString tree-sha)) + (.setRecursive true)) + _ (.next tw) + blob-sha (.name (.getObjectId tw 0))] + (.close tw) + (is (= "hello world\n" (git/blob-text repo blob-sha)))))))) From ce75d5818874b34a4ea4280e10b81f2d46456365 Mon Sep 17 00:00:00 2001 From: Devin Walters Date: Wed, 17 Jun 2026 19:22:36 -0500 Subject: [PATCH 3/8] read commits and ordering via jgit --- src/datomic/codeq/git.clj | 36 +++++++++++++++++++++++++++++++++ test/datomic/codeq/git_test.clj | 25 +++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/src/datomic/codeq/git.clj b/src/datomic/codeq/git.clj index 3b47370..4b1ba03 100644 --- a/src/datomic/codeq/git.clj +++ b/src/datomic/codeq/git.clj @@ -35,3 +35,39 @@ (let [loader (.open reader (ObjectId/fromString sha))] (String. (.getBytes loader) StandardCharsets/UTF_8)) (finally (.close reader))))) + +(defn commit-shas + "[[sha short-message] ...] in import order (parents before children). + rev nil => HEAD." + [^Repository repo rev] + (let [walk (RevWalk. repo)] + (try + (let [start (.resolve repo (or rev Constants/HEAD))] + (when (nil? start) + (throw (ex-info (str "Could not resolve rev: " (or rev "HEAD")) + {:rev rev}))) + (.markStart walk (.parseCommit walk start)) + (.sort walk RevSort/TOPO) + (.sort walk RevSort/REVERSE true) + (mapv (fn [^RevCommit c] [(.name c) (.getShortMessage c)]) + (iterator-seq (.iterator walk)))) + (finally (.close walk))))) + +(defn commit-info + "Map of commit fields for the 40-char hex sha." + [^Repository repo ^String sha] + (let [walk (RevWalk. repo)] + (try + (let [^RevCommit c (.parseCommit walk (ObjectId/fromString sha)) + ^PersonIdent author (.getAuthorIdent c) + ^PersonIdent committer (.getCommitterIdent c) + parents (mapv #(.name ^RevCommit %) (.getParents c))] + {:sha sha + :msg (string/trimr (.getFullMessage c)) + :tree (.name (.getTree c)) + :parents (seq parents) + :author (.getEmailAddress author) + :authored (.getWhen author) + :committer (.getEmailAddress committer) + :committed (.getWhen committer)}) + (finally (.close walk))))) diff --git a/test/datomic/codeq/git_test.clj b/test/datomic/codeq/git_test.clj index 9610f61..c50289a 100644 --- a/test/datomic/codeq/git_test.clj +++ b/test/datomic/codeq/git_test.clj @@ -49,3 +49,28 @@ blob-sha (.name (.getObjectId tw 0))] (.close tw) (is (= "hello world\n" (git/blob-text repo blob-sha)))))))) + +(deftest commit-info-parses-fields + (testing "commit-info returns author email, message, tree, and parents" + (with-temp-repo + (fn [dir git ^Repository repo] + (let [c1 (commit-file! git dir "a.txt" "one\n" "first") + c2 (commit-file! git dir "b.txt" "two\n" "second") + info1 (git/commit-info repo (.name c1)) + info2 (git/commit-info repo (.name c2))] + (is (= (.name c1) (:sha info1))) + (is (= "first" (:msg info1))) + (is (= "ada@example.com" (:author info1))) + (is (= "ada@example.com" (:committer info1))) + (is (instance? java.util.Date (:authored info1))) + (is (nil? (:parents info1))) + (is (= [(.name c1)] (vec (:parents info2))))))))) + +(deftest commit-shas-orders-parents-first + (testing "commit-shas returns commits parents-before-children" + (with-temp-repo + (fn [dir git ^Repository repo] + (let [c1 (commit-file! git dir "a.txt" "one\n" "first") + c2 (commit-file! git dir "a.txt" "one\ntwo\n" "second") + shas (mapv first (git/commit-shas repo nil))] + (is (= [(.name c1) (.name c2)] shas))))))) From e11ffcf6d0a6d4db87e24ee16dd54cb389ad0f69 Mon Sep 17 00:00:00 2001 From: Devin Walters Date: Wed, 17 Jun 2026 19:24:46 -0500 Subject: [PATCH 4/8] read trees via jgit --- src/datomic/codeq/git.clj | 21 +++++++++++++++++++++ test/datomic/codeq/git_test.clj | 18 ++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/datomic/codeq/git.clj b/src/datomic/codeq/git.clj index 4b1ba03..6a30a08 100644 --- a/src/datomic/codeq/git.clj +++ b/src/datomic/codeq/git.clj @@ -53,6 +53,27 @@ (iterator-seq (.iterator walk)))) (finally (.close walk))))) +(defn- mode->type [^FileMode m] + (cond + (= m FileMode/TREE) :tree + (= m FileMode/GITLINK) :commit + :else :blob)) + +(defn tree-entries + "[[sha :type filename] ...] for the direct children of tree-sha, in git tree order." + [^Repository repo ^String tree-sha] + (let [tw (doto (TreeWalk. repo) + (.addTree (ObjectId/fromString tree-sha)) + (.setRecursive false))] + (try + (loop [acc []] + (if (.next tw) + (recur (conj acc [(.name (.getObjectId tw 0)) + (mode->type (.getFileMode tw 0)) + (.getNameString tw)])) + acc)) + (finally (.close tw))))) + (defn commit-info "Map of commit fields for the 40-char hex sha." [^Repository repo ^String sha] diff --git a/test/datomic/codeq/git_test.clj b/test/datomic/codeq/git_test.clj index c50289a..b100397 100644 --- a/test/datomic/codeq/git_test.clj +++ b/test/datomic/codeq/git_test.clj @@ -74,3 +74,21 @@ c2 (commit-file! git dir "a.txt" "one\ntwo\n" "second") shas (mapv first (git/commit-shas repo nil))] (is (= [(.name c1) (.name c2)] shas))))))) + +(deftest tree-entries-lists-direct-children + (testing "tree-entries returns [sha type filename] for top-level entries" + (with-temp-repo + (fn [dir git ^Repository repo] + (.mkdirs (clojure.java.io/file dir "sub")) + (spit (clojure.java.io/file dir "sub" "nested.txt") "n\n") + (let [rc (commit-file! git dir "top.txt" "t\n" "mixed tree") + ;; also stage the nested dir + _ (.. git add (addFilepattern "sub") (call)) + rc2 (.. git commit (setMessage "with sub") + (setAuthor "Ada Lovelace" "ada@example.com") + (setCommitter "Ada Lovelace" "ada@example.com") (call)) + entries (git/tree-entries repo (.name (.getTree rc2))) + by-name (into {} (map (fn [[sha type fname]] [fname [type]]) entries))] + (is (= [:blob] (by-name "top.txt"))) + (is (= [:tree] (by-name "sub"))) + (is (every? #(= 40 (count (first %))) entries))))))) From 83d531ad2f476c0c1835944b4466b5a1311f2b61 Mon Sep 17 00:00:00 2001 From: Devin Walters Date: Wed, 17 Jun 2026 19:26:58 -0500 Subject: [PATCH 5/8] read repo uri via jgit --- src/datomic/codeq/git.clj | 14 ++++++++++++++ test/datomic/codeq/git_test.clj | 9 +++++++++ 2 files changed, 23 insertions(+) diff --git a/src/datomic/codeq/git.clj b/src/datomic/codeq/git.clj index 6a30a08..60aff56 100644 --- a/src/datomic/codeq/git.clj +++ b/src/datomic/codeq/git.clj @@ -74,6 +74,20 @@ acc)) (finally (.close tw))))) +(defn repo-uri + "[uri name] from remote.origin.url." + [^Repository repo] + (let [^String uri (.getString (.getConfig repo) "remote" "origin" "url")] + (assert uri "Can't find remote origin") + (let [noff (.lastIndexOf uri "/") + noff (if (pos? noff) noff (.lastIndexOf uri ":")) + name (subs uri (inc noff)) + _ (assert (pos? (count name)) "Can't find remote origin") + name (if (.endsWith name ".git") + (subs name 0 (.indexOf name ".")) + name)] + [uri name]))) + (defn commit-info "Map of commit fields for the 40-char hex sha." [^Repository repo ^String sha] diff --git a/test/datomic/codeq/git_test.clj b/test/datomic/codeq/git_test.clj index b100397..4a09ad8 100644 --- a/test/datomic/codeq/git_test.clj +++ b/test/datomic/codeq/git_test.clj @@ -75,6 +75,15 @@ shas (mapv first (git/commit-shas repo nil))] (is (= [(.name c1) (.name c2)] shas))))))) +(deftest repo-uri-reads-origin + (testing "repo-uri returns [uri name] from remote.origin.url" + (with-temp-repo + (fn [dir git ^Repository repo] + (let [cfg (.getConfig repo)] + (.setString cfg "remote" "origin" "url" "git@github.com:devn/codeq.git") + (.save cfg)) + (is (= ["git@github.com:devn/codeq.git" "codeq"] (git/repo-uri repo))))))) + (deftest tree-entries-lists-direct-children (testing "tree-entries returns [sha type filename] for top-level entries" (with-temp-repo From db6ec710ef3b713aeaa1c3f86a088a4573c557fb Mon Sep 17 00:00:00 2001 From: Devin Walters Date: Wed, 17 Jun 2026 19:31:31 -0500 Subject: [PATCH 6/8] import via jgit, drop git subprocess --- src/datomic/codeq/core.clj | 102 ++++++------------------------------- 1 file changed, 16 insertions(+), 86 deletions(-) diff --git a/src/datomic/codeq/core.clj b/src/datomic/codeq/core.clj index 9300e0f..fcd6731 100644 --- a/src/datomic/codeq/core.clj +++ b/src/datomic/codeq/core.clj @@ -8,13 +8,12 @@ (ns datomic.codeq.core (:require [datomic.api :as d] - [clojure.java.io :as io] [clojure.set] [clojure.string :as string] [datomic.codeq.util :refer [index->id-fn tempid?]] + [datomic.codeq.git :as git] [datomic.codeq.analyzer :as az] [datomic.codeq.analyzers.clj]) - (:import java.util.Date) (:gen-class)) (set! *warn-on-reflection* true) @@ -250,13 +249,6 @@ :db.install/_attribute :db.part/db} ]) -(defn ^java.io.Reader exec-stream - [^String cmd] - (-> (Runtime/getRuntime) - (.exec cmd) - .getInputStream - io/reader)) - (defn ensure-schema [conn] (or (-> conn d/db (d/entid :tx/commit)) @(d/transact conn schema))) @@ -293,63 +285,11 @@ ;; Push URL: https://github.com/Datomic/codeq.git ;; HEAD branch: (not queried) -(defn get-repo-uri - "returns [uri name]" - [] - (with-open [s (exec-stream (str "git remote show -n origin"))] - (let [es (line-seq s) - ^String line (second es) - uri (subs line (inc (.lastIndexOf line " "))) - noff (.lastIndexOf uri "/") - noff (if (not (pos? noff)) (.lastIndexOf uri ":") noff) - name (subs uri (inc noff)) - _ (assert (pos? (count name)) "Can't find remote origin") - name (if (.endsWith name ".git") (subs name 0 (.indexOf name ".")) name)] - [uri name]))) - -(defn dir - "Returns [[sha :type filename] ...]" - [tree] - (with-open [s (exec-stream (str "git cat-file -p " tree))] - (let [es (line-seq s)] - (mapv #(let [ss (string/split ^String % #"\s")] - [(nth ss 2) - (keyword (nth ss 1)) - (subs % (inc (.indexOf ^String % "\t")) (count %))]) - es)))) - -(defn commit - [[sha _]] - (let [trim-email (fn [s] (subs s 1 (dec (count s)))) - dt (fn [ds] (Date. (* 1000 (Integer/parseInt ds)))) - [tree parents author committer msg] - (with-open [s (exec-stream (str "git cat-file -p " sha))] - (let [lines (line-seq s) - slines (mapv #(string/split % #"\s") lines) - tree (-> slines (nth 0) (nth 1)) - [plines xs] (split-with #(= (nth % 0) "parent") (rest slines))] - [tree - (seq (map second plines)) - (vec (reverse (first xs))) - (vec (reverse (second xs))) - (->> lines - (drop-while #(not= % "")) - rest - (interpose "\n") - (apply str))]))] - {:sha sha - :msg msg - :tree tree - :parents parents - :author (trim-email (author 2)) - :authored (dt (author 1)) - :committer (trim-email (committer 2)) - :committed (dt (committer 1))})) (defn commit-tx-data - [db repo repo-name {:keys [sha msg tree parents author authored committer committed] :as commit}] + [grepo db repo repo-name {:keys [sha msg tree parents author authored committer committed] :as commit}] (let [tempid? map? ;;todo - better pred sha->id (index->id-fn db :git/sha) email->id (index->id-fn db :email/address) @@ -379,7 +319,7 @@ newpath (conj [:db/add nodeid :node/paths pathid]) (tempid? id) (conj {:db/id id :git/sha sha :git/type type})) data (if (and newpath (= type :tree)) - (let [es (dir sha)] + (let [es (git/tree-entries grepo sha)] (reduce (fn [data child] (let [[cid cdata] (f (str path "/") child) data (into data cdata)] @@ -419,19 +359,8 @@ (conj [:db/add committerid :email/address committer]))] tx)) -(defn commits - "Returns log as [[sha msg] ...], in commit order. commit-name may be nil - or any acceptable commit name arg for git log" - [commit-name] - (let [commits (with-open [s (exec-stream (str "git log --pretty=oneline --date-order --reverse " commit-name))] - (mapv - #(vector (subs % 0 40) - (subs % 41 (count %))) - (line-seq s)))] - commits)) - (defn unimported-commits - [db commit-name] + [grepo db commit-name] (let [imported (into {} (d/q '[:find ?sha ?e :where @@ -439,7 +368,8 @@ [?tx :tx/commit ?e] [?e :git/sha ?sha]] db))] - (pmap commit (remove (fn [[sha _]] (imported sha)) (commits commit-name))))) + (map (fn [[sha _]] (git/commit-info grepo sha)) + (remove (fn [[sha _]] (imported sha)) (git/commit-shas grepo commit-name))))) (defn ensure-db [db-uri] @@ -449,7 +379,7 @@ conn)) (defn import-git - [conn repo-uri repo-name commits] + [grepo conn repo-uri repo-name commits] ;;todo - add already existing commits to new repo if it includes them (println "Importing repo:" repo-uri "as:" repo-name) (let [db (d/db conn) @@ -463,14 +393,14 @@ (doseq [commit commits] (let [db (d/db conn)] (println "Importing commit:" (:sha commit)) - (d/transact conn (commit-tx-data db repo repo-name commit)))) + (d/transact conn (commit-tx-data grepo db repo repo-name commit)))) (d/request-index conn) (println "Import complete!"))) (def analyzers [(datomic.codeq.analyzers.clj/impl)]) (defn run-analyzers - [conn] + [grepo conn] (println "Analyzing...") (doseq [a analyzers] (let [aname (az/keyname a) @@ -510,8 +440,7 @@ ;;analyze them (println "analyzing file:" f " - sha: " (:git/sha (d/entity db f))) (let [db (d/db conn) - src (with-open [s (exec-stream (str "git cat-file -p " (:git/sha (d/entity db f))))] - (slurp s)) + src (git/blob-text grepo (:git/sha (d/entity db f))) adata (try (az/analyze a db f src) (catch Exception ex @@ -527,12 +456,13 @@ (defn main [& [db-uri commit]] (if db-uri + (with-open [grepo (git/open-repo)] (let [conn (ensure-db db-uri) - [repo-uri repo-name] (get-repo-uri)] - ;;(prn repo-uri) - (import-git conn repo-uri repo-name (unimported-commits (d/db conn) commit)) - (run-analyzers conn)) - (println "Usage: datomic.codeq.core db-uri [commit-name]"))) + [repo-uri repo-name] (git/repo-uri grepo)] + (import-git grepo conn repo-uri repo-name + (unimported-commits grepo (d/db conn) commit)) + (run-analyzers grepo conn))) + (println "Usage: datomic.codeq.core db-uri [commit-name]"))) (defn -main [& args] From e9cf28625148bec72eac7a4844c1eaa3688c35e1 Mon Sep 17 00:00:00 2001 From: Devin Walters Date: Wed, 17 Jun 2026 19:34:15 -0500 Subject: [PATCH 7/8] time the import and analysis --- src/datomic/codeq/core.clj | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/datomic/codeq/core.clj b/src/datomic/codeq/core.clj index fcd6731..1615f08 100644 --- a/src/datomic/codeq/core.clj +++ b/src/datomic/codeq/core.clj @@ -382,7 +382,8 @@ [grepo conn repo-uri repo-name commits] ;;todo - add already existing commits to new repo if it includes them (println "Importing repo:" repo-uri "as:" repo-name) - (let [db (d/db conn) + (let [start (System/nanoTime) + db (d/db conn) repo (or (ffirst (d/q '[:find ?e :in $ ?uri :where [?e :repo/uri ?uri]] db repo-uri)) (let [temp (d/tempid :db.part/user) @@ -395,13 +396,15 @@ (println "Importing commit:" (:sha commit)) (d/transact conn (commit-tx-data grepo db repo repo-name commit)))) (d/request-index conn) - (println "Import complete!"))) + (println "Import complete!" + (format "(%.1fs)" (/ (- (System/nanoTime) start) 1e9))))) (def analyzers [(datomic.codeq.analyzers.clj/impl)]) (defn run-analyzers [grepo conn] (println "Analyzing...") + (let [start (System/nanoTime)] (doseq [a analyzers] (let [aname (az/keyname a) exts (az/extensions a) @@ -452,7 +455,8 @@ :tx/file f :tx/analyzer aname :tx/analyzerRev arev}))))))) - (println "Analysis complete!")) + (println "Analysis complete!" + (format "(%.1fs)" (/ (- (System/nanoTime) start) 1e9))))) (defn main [& [db-uri commit]] (if db-uri From b2342b57033c575ac3e3fde5f8ebb71833af2461 Mon Sep 17 00:00:00 2001 From: Devin Walters <10421+devn@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:22:39 -0500 Subject: [PATCH 8/8] Revert "Jgit import" (#21) * Revert "time the import and analysis" This reverts commit e9cf28625148bec72eac7a4844c1eaa3688c35e1. * Revert "import via jgit, drop git subprocess" This reverts commit db6ec710ef3b713aeaa1c3f86a088a4573c557fb. * Revert "read repo uri via jgit" This reverts commit 83d531ad2f476c0c1835944b4466b5a1311f2b61. * Revert "read trees via jgit" This reverts commit e11ffcf6d0a6d4db87e24ee16dd54cb389ad0f69. * Revert "read commits and ordering via jgit" This reverts commit ce75d5818874b34a4ea4280e10b81f2d46456365. * Revert "add jgit git layer with blob reading" This reverts commit bcddc1c4e88460e9ad0ee79431333786b70a83a8. * Revert "Modernize with datomic free, deps.edn" This reverts commit 39593d8da98a509fc0cd5338167fdd68cae9abf6. --- .gitignore | 8 +- README.md | 22 +++--- build.clj | 21 ----- deps.edn | 13 --- project.clj | 11 +++ src/datomic/codeq/analyzer.clj | 7 +- src/datomic/codeq/core.clj | 114 +++++++++++++++++++++------ src/datomic/codeq/git.clj | 108 ------------------------- test/datomic/codeq/analyzer_test.clj | 11 --- test/datomic/codeq/git_test.clj | 103 ------------------------ 10 files changed, 122 insertions(+), 296 deletions(-) delete mode 100644 build.clj delete mode 100644 deps.edn create mode 100644 project.clj delete mode 100644 src/datomic/codeq/git.clj delete mode 100644 test/datomic/codeq/analyzer_test.clj delete mode 100644 test/datomic/codeq/git_test.clj diff --git a/.gitignore b/.gitignore index 215f475..cc989ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ /target +/lib /classes -.cpcache/ +/checkouts +/tmp pom.xml *.jar *.class +.lein-deps-sum +.lein-failures +.lein-plugins +*.tar diff --git a/README.md b/README.md index 4153759..93b5100 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,17 @@ ## Usage -Clone the **codeq** repo. Then, with the [Clojure CLI](https://clojure.org/guides/install_clojure) installed: +Clone the **codeq** repo. Then (in it) run: -Build a standalone jar: + lein uberjar - clojure -T:build uber +Get [Datomic Free](http://www.datomic.com/get-datomic.html) -`codeq` uses [Datomic Pro](https://www.datomic.com/) (free, no license required) via the Peer API. For a quick run with no transactor, use the in-memory `datomic:mem://` storage: +Unzip it, then start the Datomic Free transactor. Follow the instructions for [running the transactor with the free storage protocol](http://docs.datomic.com/getting-started.html) cd theGitRepoYouWantToImport - java -jar whereverYouPutCodeq/target/codeq.jar datomic:mem://git -For development without building a jar: - - clojure -M:run datomic:mem://git - -For persistent storage, start a Datomic Pro transactor and pass its URI (e.g. `datomic:dev://localhost:4334/git`) instead of `datomic:mem://`. + java -server -Xmx1g -jar whereverYouPutCodeq/target/codeq-0.1.0-SNAPSHOT-standalone.jar datomic:free://localhost:4334/git This will create a db called `git` (you can call it whatever you like) and import the commits from the local view of the repo. You should see output like: @@ -43,7 +38,12 @@ The import is not too peppy, since it shells to `git` relentlessly, but it impor You can import more than one repo into the same db. You can re-import later after some more commits and they will be incrementally added. -You can then (or during) connect to the same db URI with a peer. Note that the in-memory `datomic:mem://` db only lives inside the importing process — to browse a db after import, run against a persistent storage URI backed by a Datomic Pro transactor (e.g. `datomic:dev://localhost:4334/git`). +You can then (or during) connect to the same db URI with a peer. Or, just start the [Datomic REST service](http://docs.datomic.com/rest.html) and poke around: + + cd whereverYouPutDatomicFree + bin/rest -p 8080 free datomic:free://localhost:4334/ + +Browse to [localhost:8080/data/](http://localhost:8080/data/). You should see the `free` storage and the `git` db within it. The [schema diagram](https://github.com/downloads/Datomic/codeq/codeq.pdf) will help you get oriented. diff --git a/build.clj b/build.clj deleted file mode 100644 index 01b2f9a..0000000 --- a/build.clj +++ /dev/null @@ -1,21 +0,0 @@ -(ns build - (:require [clojure.tools.build.api :as b])) - -(def class-dir "target/classes") -(def uber-file "target/codeq.jar") -(def basis (delay (b/create-basis {:project "deps.edn"}))) - -(defn clean [_] - (b/delete {:path "target"})) - -(defn uber [_] - (clean nil) - (b/copy-dir {:src-dirs ["src"] - :target-dir class-dir}) - (b/compile-clj {:basis @basis - :ns-compile '[datomic.codeq.core] - :class-dir class-dir}) - (b/uber {:class-dir class-dir - :uber-file uber-file - :basis @basis - :main 'datomic.codeq.core})) diff --git a/deps.edn b/deps.edn deleted file mode 100644 index 0e47992..0000000 --- a/deps.edn +++ /dev/null @@ -1,13 +0,0 @@ -{:paths ["src"] - :deps {org.clojure/clojure {:mvn/version "1.12.1"} - com.datomic/peer {:mvn/version "1.0.7469" - :exclusions [org.clojure/clojure]} - org.eclipse.jgit/org.eclipse.jgit {:mvn/version "6.10.0.202406032230-r"}} - :aliases - {:run {:main-opts ["-m" "datomic.codeq.core"]} - :test {:extra-paths ["test"] - :extra-deps {io.github.cognitect-labs/test-runner - {:git/tag "v0.5.1" :git/sha "dfb30dd"}} - :main-opts ["-m" "cognitect.test-runner"]} - :build {:deps {io.github.clojure/tools.build {:mvn/version "0.10.5"}} - :ns-default build}}} diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..1ce950d --- /dev/null +++ b/project.clj @@ -0,0 +1,11 @@ +(defproject datomic/codeq "0.1.0-SNAPSHOT" + :description "codeq does a code-aware import of your git repo into a Datomic db" + :url "http://datomic.com" + :license {:name "Eclipse Public License" + :url "http://www.eclipse.org/legal/epl-v10.html"} + :main datomic.codeq.core + :plugins [[lein-tar "1.1.0"]] + :dependencies [[com.datomic/datomic-free "0.9.4699"] + [commons-codec "1.7"] + [org.clojure/clojure "1.5.1"]] + :source-paths ["src" "examples/src"]) diff --git a/src/datomic/codeq/analyzer.clj b/src/datomic/codeq/analyzer.clj index a746aca..4d662eb 100644 --- a/src/datomic/codeq/analyzer.clj +++ b/src/datomic/codeq/analyzer.clj @@ -7,7 +7,8 @@ ;; You must not remove this notice, or any other, from this software. (ns datomic.codeq.analyzer - (:import [java.io StringReader])) + (:import [java.io StringReader] + [org.apache.commons.codec.digest DigestUtils])) (set! *warn-on-reflection* true) @@ -21,9 +22,7 @@ (defn sha "Returns the hex string of the sha1 of s" [^String s] - (let [bytes (.digest (java.security.MessageDigest/getInstance "SHA-1") - (.getBytes s "UTF-8"))] - (apply str (map #(format "%02x" %) bytes)))) + (org.apache.commons.codec.digest.DigestUtils/shaHex s)) (defn ws-minify "Consecutive ws becomes a single space, then trim" diff --git a/src/datomic/codeq/core.clj b/src/datomic/codeq/core.clj index 1615f08..ef83b45 100644 --- a/src/datomic/codeq/core.clj +++ b/src/datomic/codeq/core.clj @@ -8,12 +8,13 @@ (ns datomic.codeq.core (:require [datomic.api :as d] + [clojure.java.io :as io] [clojure.set] [clojure.string :as string] [datomic.codeq.util :refer [index->id-fn tempid?]] - [datomic.codeq.git :as git] [datomic.codeq.analyzer :as az] [datomic.codeq.analyzers.clj]) + (:import java.util.Date) (:gen-class)) (set! *warn-on-reflection* true) @@ -249,6 +250,13 @@ :db.install/_attribute :db.part/db} ]) +(defn ^java.io.Reader exec-stream + [^String cmd] + (-> (Runtime/getRuntime) + (.exec cmd) + .getInputStream + io/reader)) + (defn ensure-schema [conn] (or (-> conn d/db (d/entid :tx/commit)) @(d/transact conn schema))) @@ -285,11 +293,63 @@ ;; Push URL: https://github.com/Datomic/codeq.git ;; HEAD branch: (not queried) +(defn get-repo-uri + "returns [uri name]" + [] + (with-open [s (exec-stream (str "git remote show -n origin"))] + (let [es (line-seq s) + ^String line (second es) + uri (subs line (inc (.lastIndexOf line " "))) + noff (.lastIndexOf uri "/") + noff (if (not (pos? noff)) (.lastIndexOf uri ":") noff) + name (subs uri (inc noff)) + _ (assert (pos? (count name)) "Can't find remote origin") + name (if (.endsWith name ".git") (subs name 0 (.indexOf name ".")) name)] + [uri name]))) + +(defn dir + "Returns [[sha :type filename] ...]" + [tree] + (with-open [s (exec-stream (str "git cat-file -p " tree))] + (let [es (line-seq s)] + (mapv #(let [ss (string/split ^String % #"\s")] + [(nth ss 2) + (keyword (nth ss 1)) + (subs % (inc (.indexOf ^String % "\t")) (count %))]) + es)))) + +(defn commit + [[sha _]] + (let [trim-email (fn [s] (subs s 1 (dec (count s)))) + dt (fn [ds] (Date. (* 1000 (Integer/parseInt ds)))) + [tree parents author committer msg] + (with-open [s (exec-stream (str "git cat-file -p " sha))] + (let [lines (line-seq s) + slines (mapv #(string/split % #"\s") lines) + tree (-> slines (nth 0) (nth 1)) + [plines xs] (split-with #(= (nth % 0) "parent") (rest slines))] + [tree + (seq (map second plines)) + (vec (reverse (first xs))) + (vec (reverse (second xs))) + (->> lines + (drop-while #(not= % "")) + rest + (interpose "\n") + (apply str))]))] + {:sha sha + :msg msg + :tree tree + :parents parents + :author (trim-email (author 2)) + :authored (dt (author 1)) + :committer (trim-email (committer 2)) + :committed (dt (committer 1))})) (defn commit-tx-data - [grepo db repo repo-name {:keys [sha msg tree parents author authored committer committed] :as commit}] + [db repo repo-name {:keys [sha msg tree parents author authored committer committed] :as commit}] (let [tempid? map? ;;todo - better pred sha->id (index->id-fn db :git/sha) email->id (index->id-fn db :email/address) @@ -319,7 +379,7 @@ newpath (conj [:db/add nodeid :node/paths pathid]) (tempid? id) (conj {:db/id id :git/sha sha :git/type type})) data (if (and newpath (= type :tree)) - (let [es (git/tree-entries grepo sha)] + (let [es (dir sha)] (reduce (fn [data child] (let [[cid cdata] (f (str path "/") child) data (into data cdata)] @@ -359,8 +419,19 @@ (conj [:db/add committerid :email/address committer]))] tx)) +(defn commits + "Returns log as [[sha msg] ...], in commit order. commit-name may be nil + or any acceptable commit name arg for git log" + [commit-name] + (let [commits (with-open [s (exec-stream (str "git log --pretty=oneline --date-order --reverse " commit-name))] + (mapv + #(vector (subs % 0 40) + (subs % 41 (count %))) + (line-seq s)))] + commits)) + (defn unimported-commits - [grepo db commit-name] + [db commit-name] (let [imported (into {} (d/q '[:find ?sha ?e :where @@ -368,8 +439,7 @@ [?tx :tx/commit ?e] [?e :git/sha ?sha]] db))] - (map (fn [[sha _]] (git/commit-info grepo sha)) - (remove (fn [[sha _]] (imported sha)) (git/commit-shas grepo commit-name))))) + (pmap commit (remove (fn [[sha _]] (imported sha)) (commits commit-name))))) (defn ensure-db [db-uri] @@ -379,11 +449,10 @@ conn)) (defn import-git - [grepo conn repo-uri repo-name commits] + [conn repo-uri repo-name commits] ;;todo - add already existing commits to new repo if it includes them (println "Importing repo:" repo-uri "as:" repo-name) - (let [start (System/nanoTime) - db (d/db conn) + (let [db (d/db conn) repo (or (ffirst (d/q '[:find ?e :in $ ?uri :where [?e :repo/uri ?uri]] db repo-uri)) (let [temp (d/tempid :db.part/user) @@ -394,17 +463,15 @@ (doseq [commit commits] (let [db (d/db conn)] (println "Importing commit:" (:sha commit)) - (d/transact conn (commit-tx-data grepo db repo repo-name commit)))) + (d/transact conn (commit-tx-data db repo repo-name commit)))) (d/request-index conn) - (println "Import complete!" - (format "(%.1fs)" (/ (- (System/nanoTime) start) 1e9))))) + (println "Import complete!"))) (def analyzers [(datomic.codeq.analyzers.clj/impl)]) (defn run-analyzers - [grepo conn] + [conn] (println "Analyzing...") - (let [start (System/nanoTime)] (doseq [a analyzers] (let [aname (az/keyname a) exts (az/extensions a) @@ -443,7 +510,8 @@ ;;analyze them (println "analyzing file:" f " - sha: " (:git/sha (d/entity db f))) (let [db (d/db conn) - src (git/blob-text grepo (:git/sha (d/entity db f))) + src (with-open [s (exec-stream (str "git cat-file -p " (:git/sha (d/entity db f))))] + (slurp s)) adata (try (az/analyze a db f src) (catch Exception ex @@ -455,18 +523,16 @@ :tx/file f :tx/analyzer aname :tx/analyzerRev arev}))))))) - (println "Analysis complete!" - (format "(%.1fs)" (/ (- (System/nanoTime) start) 1e9))))) + (println "Analysis complete!")) (defn main [& [db-uri commit]] (if db-uri - (with-open [grepo (git/open-repo)] (let [conn (ensure-db db-uri) - [repo-uri repo-name] (git/repo-uri grepo)] - (import-git grepo conn repo-uri repo-name - (unimported-commits grepo (d/db conn) commit)) - (run-analyzers grepo conn))) - (println "Usage: datomic.codeq.core db-uri [commit-name]"))) + [repo-uri repo-name] (get-repo-uri)] + ;;(prn repo-uri) + (import-git conn repo-uri repo-name (unimported-commits (d/db conn) commit)) + (run-analyzers conn)) + (println "Usage: datomic.codeq.core db-uri [commit-name]"))) (defn -main [& args] @@ -477,7 +543,7 @@ (comment (def uri "datomic:mem://git") -;;(def uri "datomic:dev://localhost:4334/git") +;;(def uri "datomic:free://localhost:4334/git") (datomic.codeq.core/main uri "c3bd979cfe65da35253b25cb62aad4271430405c") (datomic.codeq.core/main uri "20f8db11804afc8c5a1752257d5fdfcc2d131d08") (datomic.codeq.core/main uri) diff --git a/src/datomic/codeq/git.clj b/src/datomic/codeq/git.clj deleted file mode 100644 index 60aff56..0000000 --- a/src/datomic/codeq/git.clj +++ /dev/null @@ -1,108 +0,0 @@ -;; Copyright (c) Metadata Partners, LLC and Contributors. All rights reserved. -;; The use and distribution terms for this software are covered by the -;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) -;; which can be found in the file epl-v10.html at the root of this distribution. -;; By using this software in any fashion, you are agreeing to be bound by -;; the terms of this license. -;; You must not remove this notice, or any other, from this software. - -(ns datomic.codeq.git - (:require [clojure.string :as string]) - (:import [java.io File] - [java.nio.charset StandardCharsets] - [org.eclipse.jgit.lib Repository ObjectId FileMode PersonIdent Constants] - [org.eclipse.jgit.storage.file FileRepositoryBuilder] - [org.eclipse.jgit.revwalk RevWalk RevCommit RevSort] - [org.eclipse.jgit.treewalk TreeWalk])) - -(set! *warn-on-reflection* true) - -(defn ^Repository open-repo - "Open the git repository discovered from dir (defaults to cwd)." - ([] (open-repo (System/getProperty "user.dir"))) - ([dir] - (-> (FileRepositoryBuilder.) - (.findGitDir (File. ^String (str dir))) - (.setMustExist true) - (.readEnvironment) - (.build)))) - -(defn ^String blob-text - "UTF-8 text of the blob named by the 40-char hex sha." - [^Repository repo ^String sha] - (let [reader (.newObjectReader repo)] - (try - (let [loader (.open reader (ObjectId/fromString sha))] - (String. (.getBytes loader) StandardCharsets/UTF_8)) - (finally (.close reader))))) - -(defn commit-shas - "[[sha short-message] ...] in import order (parents before children). - rev nil => HEAD." - [^Repository repo rev] - (let [walk (RevWalk. repo)] - (try - (let [start (.resolve repo (or rev Constants/HEAD))] - (when (nil? start) - (throw (ex-info (str "Could not resolve rev: " (or rev "HEAD")) - {:rev rev}))) - (.markStart walk (.parseCommit walk start)) - (.sort walk RevSort/TOPO) - (.sort walk RevSort/REVERSE true) - (mapv (fn [^RevCommit c] [(.name c) (.getShortMessage c)]) - (iterator-seq (.iterator walk)))) - (finally (.close walk))))) - -(defn- mode->type [^FileMode m] - (cond - (= m FileMode/TREE) :tree - (= m FileMode/GITLINK) :commit - :else :blob)) - -(defn tree-entries - "[[sha :type filename] ...] for the direct children of tree-sha, in git tree order." - [^Repository repo ^String tree-sha] - (let [tw (doto (TreeWalk. repo) - (.addTree (ObjectId/fromString tree-sha)) - (.setRecursive false))] - (try - (loop [acc []] - (if (.next tw) - (recur (conj acc [(.name (.getObjectId tw 0)) - (mode->type (.getFileMode tw 0)) - (.getNameString tw)])) - acc)) - (finally (.close tw))))) - -(defn repo-uri - "[uri name] from remote.origin.url." - [^Repository repo] - (let [^String uri (.getString (.getConfig repo) "remote" "origin" "url")] - (assert uri "Can't find remote origin") - (let [noff (.lastIndexOf uri "/") - noff (if (pos? noff) noff (.lastIndexOf uri ":")) - name (subs uri (inc noff)) - _ (assert (pos? (count name)) "Can't find remote origin") - name (if (.endsWith name ".git") - (subs name 0 (.indexOf name ".")) - name)] - [uri name]))) - -(defn commit-info - "Map of commit fields for the 40-char hex sha." - [^Repository repo ^String sha] - (let [walk (RevWalk. repo)] - (try - (let [^RevCommit c (.parseCommit walk (ObjectId/fromString sha)) - ^PersonIdent author (.getAuthorIdent c) - ^PersonIdent committer (.getCommitterIdent c) - parents (mapv #(.name ^RevCommit %) (.getParents c))] - {:sha sha - :msg (string/trimr (.getFullMessage c)) - :tree (.name (.getTree c)) - :parents (seq parents) - :author (.getEmailAddress author) - :authored (.getWhen author) - :committer (.getEmailAddress committer) - :committed (.getWhen committer)}) - (finally (.close walk))))) diff --git a/test/datomic/codeq/analyzer_test.clj b/test/datomic/codeq/analyzer_test.clj deleted file mode 100644 index 21bae11..0000000 --- a/test/datomic/codeq/analyzer_test.clj +++ /dev/null @@ -1,11 +0,0 @@ -(ns datomic.codeq.analyzer-test - (:require [clojure.test :refer [deftest testing is]] - [datomic.codeq.analyzer :as az])) - -(deftest sha-matches-known-sha1 - (testing "sha returns lowercase hex sha1 matching commons-codec shaHex output" - ;; Known SHA-1 values (lowercase hex), verifiable via `printf '%s' "..." | shasum` - (is (= "da39a3ee5e6b4b0d3255bfef95601890afd80709" (az/sha ""))) - (is (= "a9993e364706816aba3e25717850c26c9cd0d89d" (az/sha "abc"))) - (is (= "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33" (az/sha "foo"))) - (is (= 40 (count (az/sha "anything")))))) diff --git a/test/datomic/codeq/git_test.clj b/test/datomic/codeq/git_test.clj deleted file mode 100644 index 4a09ad8..0000000 --- a/test/datomic/codeq/git_test.clj +++ /dev/null @@ -1,103 +0,0 @@ -(ns datomic.codeq.git-test - (:require [clojure.test :refer [deftest testing is]] - [clojure.java.io :as io] - [datomic.codeq.git :as git]) - (:import [org.eclipse.jgit.api Git] - [org.eclipse.jgit.lib Repository] - [java.io File])) - -(defn- temp-dir ^File [] - (let [d (File/createTempFile "codeq-git-test" "")] - (.delete d) (.mkdirs d) d)) - -(defn- delete-recursively [^File f] - (when (.isDirectory f) - (doseq [c (.listFiles f)] (delete-recursively c))) - (.delete f)) - -(defn with-temp-repo - "Calls (f dir git repo) with a fresh initialized repo; cleans up after." - [f] - (let [dir (temp-dir) - git (.. (Git/init) (setDirectory dir) (call)) - repo (.getRepository git)] - (try (f dir git repo) - (finally (.close git) (delete-recursively dir))))) - -(defn commit-file! - "Writes fname=content, stages, commits with a deterministic author. Returns RevCommit." - [^Git git ^File dir ^String fname ^String content ^String msg] - (spit (io/file dir fname) content) - (.. git add (addFilepattern fname) (call)) - (.. git commit - (setMessage msg) - (setAuthor "Ada Lovelace" "ada@example.com") - (setCommitter "Ada Lovelace" "ada@example.com") - (call))) - -(deftest blob-text-round-trips - (testing "blob-text returns the file content for a committed blob" - (with-temp-repo - (fn [dir git ^Repository repo] - (let [rc (commit-file! git dir "hello.txt" "hello world\n" "add hello") - tree-sha (.name (.getTree rc)) - ;; find the blob sha via JGit TreeWalk in the test - tw (doto (org.eclipse.jgit.treewalk.TreeWalk. repo) - (.addTree (org.eclipse.jgit.lib.ObjectId/fromString tree-sha)) - (.setRecursive true)) - _ (.next tw) - blob-sha (.name (.getObjectId tw 0))] - (.close tw) - (is (= "hello world\n" (git/blob-text repo blob-sha)))))))) - -(deftest commit-info-parses-fields - (testing "commit-info returns author email, message, tree, and parents" - (with-temp-repo - (fn [dir git ^Repository repo] - (let [c1 (commit-file! git dir "a.txt" "one\n" "first") - c2 (commit-file! git dir "b.txt" "two\n" "second") - info1 (git/commit-info repo (.name c1)) - info2 (git/commit-info repo (.name c2))] - (is (= (.name c1) (:sha info1))) - (is (= "first" (:msg info1))) - (is (= "ada@example.com" (:author info1))) - (is (= "ada@example.com" (:committer info1))) - (is (instance? java.util.Date (:authored info1))) - (is (nil? (:parents info1))) - (is (= [(.name c1)] (vec (:parents info2))))))))) - -(deftest commit-shas-orders-parents-first - (testing "commit-shas returns commits parents-before-children" - (with-temp-repo - (fn [dir git ^Repository repo] - (let [c1 (commit-file! git dir "a.txt" "one\n" "first") - c2 (commit-file! git dir "a.txt" "one\ntwo\n" "second") - shas (mapv first (git/commit-shas repo nil))] - (is (= [(.name c1) (.name c2)] shas))))))) - -(deftest repo-uri-reads-origin - (testing "repo-uri returns [uri name] from remote.origin.url" - (with-temp-repo - (fn [dir git ^Repository repo] - (let [cfg (.getConfig repo)] - (.setString cfg "remote" "origin" "url" "git@github.com:devn/codeq.git") - (.save cfg)) - (is (= ["git@github.com:devn/codeq.git" "codeq"] (git/repo-uri repo))))))) - -(deftest tree-entries-lists-direct-children - (testing "tree-entries returns [sha type filename] for top-level entries" - (with-temp-repo - (fn [dir git ^Repository repo] - (.mkdirs (clojure.java.io/file dir "sub")) - (spit (clojure.java.io/file dir "sub" "nested.txt") "n\n") - (let [rc (commit-file! git dir "top.txt" "t\n" "mixed tree") - ;; also stage the nested dir - _ (.. git add (addFilepattern "sub") (call)) - rc2 (.. git commit (setMessage "with sub") - (setAuthor "Ada Lovelace" "ada@example.com") - (setCommitter "Ada Lovelace" "ada@example.com") (call)) - entries (git/tree-entries repo (.name (.getTree rc2))) - by-name (into {} (map (fn [[sha type fname]] [fname [type]]) entries))] - (is (= [:blob] (by-name "top.txt"))) - (is (= [:tree] (by-name "sub"))) - (is (every? #(= 40 (count (first %))) entries)))))))