flycouchdb

0.2.2-SNAPSHOT


Migration tool for CouchDB

dependencies

org.clojure/clojure
1.6.0
couchdb-extension
0.1.4
clj-http
1.0.1
ch.qos.logback/logback-classic
1.1.2
com.ashafa/clutch
0.4.0
clj-time
0.9.0
slingshot
0.12.2
org.jboss/jboss-vfs
3.1.0.Final



(this space intentionally left almost blank)
 
(ns flycouchdb.migration
  (:use [flycouchdb.parser.parse-edn-structures :only (parse-edn-structures apply-functions)]
        [flycouchdb.parser.parse-migration-names :only (generate-migrations-structure)]
        [flycouchdb.scanner :only (extract-migrations)]
        [be.dsquare.clutch :only (couch up? exist? create-view! get-view)]
        [clojure.java.io :only (input-stream)]
        [clj-time.core :only (now)]
        [slingshot.slingshot :only [throw+ try+]])
  (:require [com.ashafa.clutch :as clutch]))
(def migration-db "migration-db")
(def migration-counter (atom 0))
(defrecord FlyCouchDB [^String location-folder
                       ^String datasource-url
                       ^String datasource-user
                       ^String datasource-password])

Validates the connection

(defn- validate-connection
  []
  (let [db (couch migration-db)]
    (if (up? db)
      db
      (throw+ {:type    :flycouchdb :fn "validate-connection"
               :message (str "Trying to connect to CouchDB. Connection is down!")}))))

Creates the migration database in CouchDB in case it does not exist. The migration database is named 'migration-db', and is a hardcoded name so far.

(defn- create-if-not-exists
  []
  (let [db (couch migration-db)]
    (when-not (exist? db)
      (clutch/create! db)
      (create-view! db
        "migration-template"
        "order-migrations"
        "function(doc) {if (doc.counter) {emit(doc.counter, doc);}}"))))
(defmulti slurp-edn-structures :source)
(defmethod slurp-edn-structures :file [migration]
  (assoc migration :edn-structure (read-string (slurp (:file migration)))))
(defmethod slurp-edn-structures :jar [migration]
  (assoc migration :edn-structure (read-string (slurp (input-stream (:file migration))))))
(defmethod slurp-edn-structures :vfs [migration]
  (assoc migration :edn-structure (read-string (slurp (.openStream (:file migration))))))
(defn- columns [column-names]
  (fn [row]
    (vec (map row column-names))))

We compare by :version and subversion

(defn- compare-by-version-subversion
  [{version-a :version subversion-a :subversion}
   {version-b :version subversion-b :subversion}]
  (cond
    (< version-a version-b) -1
    (> version-a version-b) 1
    (< subversion-a subversion-b) -1
    (> subversion-a subversion-b) 1
    :else 0))

Update the counter with the last migration that was run

(defn- update-counter!
  [{counter :counter :or {:counter 0}}]
  (reset! migration-counter counter))

Getting the last migration from CouchDB, ordered by the counter entry

(defn- get-last-migration
  []
  (->
    migration-db
    couch
    (get-view "migration-template" "order-migrations")
    last
    :value
    (#(if (nil? %)
       {:version -1 :subversion -1 :counter 0}
       %))))

Start the migration process

(defn migrate
  [^FlyCouchDB flycouchdb]
  (do
    (validate-connection)
    (create-if-not-exists)
    (let [last-migration (get-last-migration)]
      (update-counter! last-migration)
      (->>
        flycouchdb
        extract-migrations
        generate-migrations-structure
        (sort-by (columns [:version :subversion]))
        (filter #(= (compare-by-version-subversion % last-migration) 1))
        (map (fn [migration] (slurp-edn-structures migration)))
        (map (fn [migration] (assoc migration :edn-function (parse-edn-structures (:edn-structure migration)))))
        (map (fn [migration] (assoc migration :ts (str (now)))))
        (map (fn [migration] (assoc migration :counter (swap! migration-counter inc))))
        (mapv (fn [migration]
                (do
                  (apply-functions migration)
                  (println (:name migration))
                  (let [db (couch migration-db)
                        edn-migration (-> migration
                                        (dissoc :file :edn-function :file :edn-structure)
                                        (assoc :dbname (:dbname (:edn-structure migration)) :action (:action (:edn-structure migration))))]
                    (clutch/assoc! db (:name migration) edn-migration)))))))))

Returns an instance of an implementation of FlyCouchDB

(defn flycouchdb
  [^String location-folder]
  (FlyCouchDB. location-folder nil nil nil))
 
(ns flycouchdb.parser.parse-edn-structures
  (:use [be.dsquare.clutch :only (couch drop! up? exist? create-view! take-all get-view get-user-view create-user-view!)]
        [com.ashafa.clutch :only (create!)]
        [slingshot.slingshot :only [throw+ try+]])
  (:require [com.ashafa.clutch :as clutch]))
(defmulti parse-edn-structures :action)
(defmethod parse-edn-structures :create [{dbname :dbname}]
  (fn [] (let [couchdb (couch dbname)]
           (create! couchdb))))
(defmethod parse-edn-structures :delete [{dbname :dbname}]
  (fn [] (let [couchdb (couch dbname)]
           (drop! couchdb))))
(defmethod parse-edn-structures :create-view
  [{dbname :dbname {:keys [view-design view-name view-function]} :create-view}]
  (fn [] (let [couchdb (couch dbname)]
           (create-view! couchdb view-design view-name view-function))))
(defmethod parse-edn-structures :rename-keys
  [{dbname :dbname {:keys [filter-fn rename-fn]} :rename-keys}]
  (fn [] (let [couchdb (couch dbname)]
           (->>
             couchdb
             take-all
             (map #(second %))
             (filter (eval filter-fn))
             (map (fn [entry]
                    {:old_id        (:_id entry)
                     :new_id        ((eval rename-fn) entry)
                     :document-body (dissoc entry :_id :_rev)}))
             (mapv (fn [{:keys [old_id new_id document-body]}]
                     (do
                       (clutch/dissoc! couchdb old_id)
                       (clutch/assoc! couchdb new_id document-body))))))))
(defmethod parse-edn-structures :edit-entries
  [{dbname :dbname {:keys [filter-fn edit-fn]} :edit-entries}]
  (fn [] (let [couchdb (couch dbname)]
           (->>
             couchdb
             take-all
             (map #(second %))
             (filter (eval filter-fn))
             (mapv (fn [entry] (clutch/assoc! couchdb (:_id entry) ((eval edit-fn) entry))))))))
(defmethod parse-edn-structures :composite
  [{{composite-fn :composite-fn} :composite}]
  (let [migration-functions (->>
                              ((eval composite-fn))
                              (mapv (fn [edn-structure] (parse-edn-structures edn-structure))))]
    (fn [] (->>
             migration-functions
             (mapv (fn [m-fn] ((eval m-fn))))))))
(defmethod parse-edn-structures :insert-documents
  [{dbname :dbname {composite-fn :insert-documents-fn} :insert-documents}]
  (let [db (couch dbname)
        documents ((eval composite-fn))]
    (fn [] (->>
             documents
             (mapv (fn [document] (clutch/assoc! db (:_id document) document)))))))

Apply the anonymous functions that were created in this namespace so that there is no problem with the imports.

(defn apply-functions
  [{:keys [edn-function edn-structure file-name ts]}]
  (edn-function))
 
(ns flycouchdb.parser.parse-migration-names
  (:use clojure.pprint
        [slingshot.slingshot :only [throw+ try+]]))

Checks if it's an edn file

(defn- edn-file?
  [file-name]
  (let [total-file (count file-name)]
    (= (subs file-name (- total-file 4) total-file) ".edn")))

Remove the extension from the file name

(defn- remove-file-extension
  ([file-name]
   (remove-file-extension file-name "edn"))
  ([file-name extension]
   (subs file-name 0 (- (count file-name) (+ 1 (count extension))))))

Validate if it's a correct edn file and retorn name

(defn- validate-edn-file
  [{file-name :file-name}]
  (if (edn-file? file-name)
    {:name (remove-file-extension file-name)}
    (throw+ {:type    :flycouchdb :fn "validate-edn-file"
             :message (str "This file: " file-name " is not a correct edn file")})))

Checks if the version number is V{d}{ddd}_ Example: V1101_ where this is gonna be the migration 101 from the first version

(defn- correct-version?
  [^String file-name]
  (let [find-version (re-find #"V(\d+)_" file-name)]
    (not (nil? find-version))))

Extract version from migration name V1_

(defn- extract-migration-number
  [^String name]
  (->>
    name
    (re-find #"V(\d+)_")
    second
    read-string))

Checks if the version number is V{d}{ddd}_ Example: V1101_ where this is gonna be the migration 101 from the first version

(defn- correct-subversion-number?
  [^String name]
  (let [find-version (re-find #"V(\d+)_(\d+)__[a-zA-Z]+" name)]
    (not (nil? find-version))))

Checks if the version number is V{d}{ddd}_ Example: V1101_ where this is gonna be the migration 101 from the first version

(defn- extract-migration-subversion-number
  [^String name]
  (->>
    name
    (re-find #"V(\d+)_(\d+)__[a-zA-Z]+")
    rest
    second
    read-string))

Validate if the version is correctly formated and returns it

(defn- validate-migration-version
  [{name :name}]
  (if (correct-version? name)
    {:version (extract-migration-number name)}
    (throw+ {:type    :flycouchdb :fn "validate-migration-version"
             :message (str "This version of " name " is not valid!"
                        "\nExample: V1_131__Create_Database.edn")})))
(defn- validate-subversion-number
  [{name :name}]
  (if (correct-subversion-number? name)
    {:subversion (extract-migration-subversion-number name)}
    (throw+ {:type    :flycouchdb :fn "validate-subversion-number"
             :message (str "This sub-version of " name " is not valid!"
                        "\nExample: V1_131__Create_Database.edn")})))

This generates the following structure per each migration: {:version 1, :subversion 132, :name 'V1132DeleteDatabase', :file #<BufferedInputStream java.io.BufferedInputStream@2b46504>, :file-name 'V1132DeleteDatabase.edn'}

(defn generate-migrations-structure
  [folder-seq]
  (->>
    folder-seq
    (map #(merge % (validate-edn-file %)))
    (map #(merge % (validate-migration-version %)))
    (map #(merge % (validate-subversion-number %)))))
 
(ns flycouchdb.scanner
  (:use [clojure.java.io :only (file resource)]
        [slingshot.slingshot :only [throw+ try+]])
  (:import [java.net URL URLDecoder]
           [java.io File]
           [java.util.jar JarFile]
           [org.jboss.vfs VFS VirtualFile]
           [clojure.lang ISeq]))

Check that the permissions are ok for reading it

(defn- validate-file-can-be-read
  [^ISeq file]
  (if (.canRead file)
    file
    (throw+ {:type    :flycouchdb :fn "validate-file-can-be-read"
             :message (str "This file: " (.getName file) " can not be read")})))

Create a map that contains all the properties from a migration file

(defn- generate-basic-structure-from-file
  [^ISeq file]
  {:file-name (.getName file)
   :file      file
   :source    :file})

validates if it's a file

(defn- validate-is-a-file
  [^ISeq file]
  (try
    (when (.isFile file)
      file)
    (catch Exception _
      (throw+ {:type    :flycouchdb :fn "validate-is-a-file"
               :message "Is not a valid File"}))))

Extracting the migrations from a file/directory resource

(defn extract-file-migrations
  [^File location-folder]
  (let [folder-seq (file-seq location-folder)]
    (->>
      folder-seq
      rest
      (map #(validate-is-a-file %))
      (map #(validate-file-can-be-read %))
      (map #(generate-basic-structure-from-file %)))))

Create a map that contains all the properties from a Virtual File System

(defn generate-basic-structure-from-vfs
  [^VirtualFile v]
  {:file-name (.getName v)
   :file      v
   :source    :vfs})

Extracting the migrations from a VFS resource

(defn- extract-vfs-migrations
  [^String location-folder]
  (->>
    location-folder
    .getPath
    (. URLDecoder decode)
    File.
    .getAbsolutePath
    (. VFS getChild)
    .getChildren
    (mapv generate-basic-structure-from-vfs)))

Create a map that contains all the properties from a JAR file

(defn- generate-basic-structure-from-jar
  [migration-folder jar-entry]
  {:file-name (subs (.getName jar-entry) (count migration-folder) (count (.getName jar-entry)))
   :file      (resource (.getName jar-entry))
   :source    :jar})

Extracting the migrations from a JAR resource

(defn- extract-jar-migrations
  [[^String jar-path ^String migration-folder]]
  (->>
    jar-path
    JarFile.
    .entries
    enumeration-seq
    (filter #(re-find (re-pattern (str migration-folder "*")) (.getName %)))
    (filter #(re-find #"\.edn{1}|\.clj{1}" (.getName %)))
    (mapv (partial generate-basic-structure-from-jar migration-folder))))

It splits the jar path and the internal folder from the jar URL from jar:file:/Users/haduart/flycouchdb-example-0.1.0-SNAPSHOT-standalone.jar!/migrations/ to ['/Users/haduart/flycouchdb-example-0.1.0-SNAPSHOT-standalone.jar' 'migrations/']

(defn- parse-jar-path-and-folder
  [^URL jar-url]
  (let [jar-file (str jar-url)
        jar-string (subs jar-file (count "jar:file:") (count jar-file))
        jar-position (.indexOf jar-string ".jar!")]
    [(str (subs jar-string 0 jar-position) ".jar")
     (subs jar-string (+ jar-position (count ".jar!/")) (count jar-string))]))

Wraps around the Java call. Just to make the function testable

(defn- get-url-protocol
  [location-folder]
  (.getProtocol location-folder))
(defmulti extract-migrations
  (fn [{location-folder :location-folder}] (get-url-protocol location-folder)))
(defmethod extract-migrations "jar" [{location-folder :location-folder}]
  (->
    location-folder
    parse-jar-path-and-folder
    extract-jar-migrations))
(defmethod extract-migrations "file" [{location-folder :location-folder}]
  (->
    location-folder
    file
    extract-file-migrations))
(defmethod extract-migrations "vfs" [{location-folder :location-folder}]
  (->
    location-folder
    extract-vfs-migrations))