dependencies
| (this space intentionally left almost blank) | ||||||||||||||||||||||||
(ns flycouchdb.migration
(:use [flycouchdb.parser.parse-edn-structures :only (parse-edn-structures apply-functions)]
[flycouchdb.parser.parse-migration-names :only (generate-migrations-structure)]
[flycouchdb.scanner :only (extract-migrations)]
[be.dsquare.clutch :only (couch up? exist? create-view! get-view)]
[clojure.java.io :only (input-stream)]
[clj-time.core :only (now)]
[slingshot.slingshot :only [throw+ try+]])
(:require [com.ashafa.clutch :as clutch])) | |||||||||||||||||||||||||
(def migration-db "migration-db") | |||||||||||||||||||||||||
(def migration-counter (atom 0)) | |||||||||||||||||||||||||
(defrecord FlyCouchDB [^String location-folder
^String datasource-url
^String datasource-user
^String datasource-password]) | |||||||||||||||||||||||||
Validates the connection | (defn- validate-connection
[]
(let [db (couch migration-db)]
(if (up? db)
db
(throw+ {:type :flycouchdb :fn "validate-connection"
:message (str "Trying to connect to CouchDB. Connection is down!")})))) | ||||||||||||||||||||||||
Creates the migration database in CouchDB in case it does not exist. The migration database is named 'migration-db', and is a hardcoded name so far. | (defn- create-if-not-exists
[]
(let [db (couch migration-db)]
(when-not (exist? db)
(clutch/create! db)
(create-view! db
"migration-template"
"order-migrations"
"function(doc) {if (doc.counter) {emit(doc.counter, doc);}}")))) | ||||||||||||||||||||||||
(defmulti slurp-edn-structures :source) | |||||||||||||||||||||||||
(defmethod slurp-edn-structures :file [migration] (assoc migration :edn-structure (read-string (slurp (:file migration))))) | |||||||||||||||||||||||||
(defmethod slurp-edn-structures :jar [migration] (assoc migration :edn-structure (read-string (slurp (input-stream (:file migration)))))) | |||||||||||||||||||||||||
(defmethod slurp-edn-structures :vfs [migration] (assoc migration :edn-structure (read-string (slurp (.openStream (:file migration)))))) | |||||||||||||||||||||||||
(defn- columns [column-names]
(fn [row]
(vec (map row column-names)))) | |||||||||||||||||||||||||
We compare by :version and subversion | (defn- compare-by-version-subversion
[{version-a :version subversion-a :subversion}
{version-b :version subversion-b :subversion}]
(cond
(< version-a version-b) -1
(> version-a version-b) 1
(< subversion-a subversion-b) -1
(> subversion-a subversion-b) 1
:else 0)) | ||||||||||||||||||||||||
Update the counter with the last migration that was run | (defn- update-counter!
[{counter :counter :or {:counter 0}}]
(reset! migration-counter counter)) | ||||||||||||||||||||||||
Getting the last migration from CouchDB, ordered by the counter entry | (defn- get-last-migration
[]
(->
migration-db
couch
(get-view "migration-template" "order-migrations")
last
:value
(#(if (nil? %)
{:version -1 :subversion -1 :counter 0}
%)))) | ||||||||||||||||||||||||
Start the migration process | (defn migrate
[^FlyCouchDB flycouchdb]
(do
(validate-connection)
(create-if-not-exists)
(let [last-migration (get-last-migration)]
(update-counter! last-migration)
(->>
flycouchdb
extract-migrations
generate-migrations-structure
(sort-by (columns [:version :subversion]))
(filter #(= (compare-by-version-subversion % last-migration) 1))
(map (fn [migration] (slurp-edn-structures migration)))
(map (fn [migration] (assoc migration :edn-function (parse-edn-structures (:edn-structure migration)))))
(map (fn [migration] (assoc migration :ts (str (now)))))
(map (fn [migration] (assoc migration :counter (swap! migration-counter inc))))
(mapv (fn [migration]
(do
(apply-functions migration)
(println (:name migration))
(let [db (couch migration-db)
edn-migration (-> migration
(dissoc :file :edn-function :file :edn-structure)
(assoc :dbname (:dbname (:edn-structure migration)) :action (:action (:edn-structure migration))))]
(clutch/assoc! db (:name migration) edn-migration))))))))) | ||||||||||||||||||||||||
Returns an instance of an implementation of FlyCouchDB | (defn flycouchdb [^String location-folder] (FlyCouchDB. location-folder nil nil nil)) | ||||||||||||||||||||||||
(ns flycouchdb.parser.parse-edn-structures
(:use [be.dsquare.clutch :only (couch drop! up? exist? create-view! take-all get-view get-user-view create-user-view!)]
[com.ashafa.clutch :only (create!)]
[slingshot.slingshot :only [throw+ try+]])
(:require [com.ashafa.clutch :as clutch])) | |||||||||||||||||||||||||
(defmulti parse-edn-structures :action) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :create [{dbname :dbname}]
(fn [] (let [couchdb (couch dbname)]
(create! couchdb)))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :delete [{dbname :dbname}]
(fn [] (let [couchdb (couch dbname)]
(drop! couchdb)))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :create-view
[{dbname :dbname {:keys [view-design view-name view-function]} :create-view}]
(fn [] (let [couchdb (couch dbname)]
(create-view! couchdb view-design view-name view-function)))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :rename-keys
[{dbname :dbname {:keys [filter-fn rename-fn]} :rename-keys}]
(fn [] (let [couchdb (couch dbname)]
(->>
couchdb
take-all
(map #(second %))
(filter (eval filter-fn))
(map (fn [entry]
{:old_id (:_id entry)
:new_id ((eval rename-fn) entry)
:document-body (dissoc entry :_id :_rev)}))
(mapv (fn [{:keys [old_id new_id document-body]}]
(do
(clutch/dissoc! couchdb old_id)
(clutch/assoc! couchdb new_id document-body)))))))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :edit-entries
[{dbname :dbname {:keys [filter-fn edit-fn]} :edit-entries}]
(fn [] (let [couchdb (couch dbname)]
(->>
couchdb
take-all
(map #(second %))
(filter (eval filter-fn))
(mapv (fn [entry] (clutch/assoc! couchdb (:_id entry) ((eval edit-fn) entry)))))))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :composite
[{{composite-fn :composite-fn} :composite}]
(let [migration-functions (->>
((eval composite-fn))
(mapv (fn [edn-structure] (parse-edn-structures edn-structure))))]
(fn [] (->>
migration-functions
(mapv (fn [m-fn] ((eval m-fn)))))))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :insert-documents
[{dbname :dbname {composite-fn :insert-documents-fn} :insert-documents}]
(let [db (couch dbname)
documents ((eval composite-fn))]
(fn [] (->>
documents
(mapv (fn [document] (clutch/assoc! db (:_id document) document))))))) | |||||||||||||||||||||||||
Apply the anonymous functions that were created in this namespace so that there is no problem with the imports. | (defn apply-functions
[{:keys [edn-function edn-structure file-name ts]}]
(edn-function)) | ||||||||||||||||||||||||
(ns flycouchdb.parser.parse-migration-names
(:use clojure.pprint
[slingshot.slingshot :only [throw+ try+]])) | |||||||||||||||||||||||||
Checks if it's an edn file | (defn- edn-file?
[file-name]
(let [total-file (count file-name)]
(= (subs file-name (- total-file 4) total-file) ".edn"))) | ||||||||||||||||||||||||
Remove the extension from the file name | (defn- remove-file-extension ([file-name] (remove-file-extension file-name "edn")) ([file-name extension] (subs file-name 0 (- (count file-name) (+ 1 (count extension)))))) | ||||||||||||||||||||||||
Validate if it's a correct edn file and retorn name | (defn- validate-edn-file
[{file-name :file-name}]
(if (edn-file? file-name)
{:name (remove-file-extension file-name)}
(throw+ {:type :flycouchdb :fn "validate-edn-file"
:message (str "This file: " file-name " is not a correct edn file")}))) | ||||||||||||||||||||||||
Checks if the version number is V{d}{ddd}_ Example: V1101_ where this is gonna be the migration 101 from the first version | (defn- correct-version?
[^String file-name]
(let [find-version (re-find #"V(\d+)_" file-name)]
(not (nil? find-version)))) | ||||||||||||||||||||||||
Extract version from migration name V1_ | (defn- extract-migration-number
[^String name]
(->>
name
(re-find #"V(\d+)_")
second
read-string)) | ||||||||||||||||||||||||
Checks if the version number is V{d}{ddd}_ Example: V1101_ where this is gonna be the migration 101 from the first version | (defn- correct-subversion-number?
[^String name]
(let [find-version (re-find #"V(\d+)_(\d+)__[a-zA-Z]+" name)]
(not (nil? find-version)))) | ||||||||||||||||||||||||
Checks if the version number is V{d}{ddd}_ Example: V1101_ where this is gonna be the migration 101 from the first version | (defn- extract-migration-subversion-number
[^String name]
(->>
name
(re-find #"V(\d+)_(\d+)__[a-zA-Z]+")
rest
second
read-string)) | ||||||||||||||||||||||||
Validate if the version is correctly formated and returns it | (defn- validate-migration-version
[{name :name}]
(if (correct-version? name)
{:version (extract-migration-number name)}
(throw+ {:type :flycouchdb :fn "validate-migration-version"
:message (str "This version of " name " is not valid!"
"\nExample: V1_131__Create_Database.edn")}))) | ||||||||||||||||||||||||
(defn- validate-subversion-number
[{name :name}]
(if (correct-subversion-number? name)
{:subversion (extract-migration-subversion-number name)}
(throw+ {:type :flycouchdb :fn "validate-subversion-number"
:message (str "This sub-version of " name " is not valid!"
"\nExample: V1_131__Create_Database.edn")}))) | |||||||||||||||||||||||||
This generates the following structure per each migration: {:version 1, :subversion 132, :name 'V1132DeleteDatabase', :file #<BufferedInputStream java.io.BufferedInputStream@2b46504>, :file-name 'V1132DeleteDatabase.edn'} | (defn generate-migrations-structure
[folder-seq]
(->>
folder-seq
(map #(merge % (validate-edn-file %)))
(map #(merge % (validate-migration-version %)))
(map #(merge % (validate-subversion-number %))))) | ||||||||||||||||||||||||
(ns flycouchdb.scanner
(:use [clojure.java.io :only (file resource)]
[slingshot.slingshot :only [throw+ try+]])
(:import [java.net URL URLDecoder]
[java.io File]
[java.util.jar JarFile]
[org.jboss.vfs VFS VirtualFile]
[clojure.lang ISeq])) | |||||||||||||||||||||||||
Check that the permissions are ok for reading it | (defn- validate-file-can-be-read
[^ISeq file]
(if (.canRead file)
file
(throw+ {:type :flycouchdb :fn "validate-file-can-be-read"
:message (str "This file: " (.getName file) " can not be read")}))) | ||||||||||||||||||||||||
Create a map that contains all the properties from a migration file | (defn- generate-basic-structure-from-file
[^ISeq file]
{:file-name (.getName file)
:file file
:source :file}) | ||||||||||||||||||||||||
validates if it's a file | (defn- validate-is-a-file
[^ISeq file]
(try
(when (.isFile file)
file)
(catch Exception _
(throw+ {:type :flycouchdb :fn "validate-is-a-file"
:message "Is not a valid File"})))) | ||||||||||||||||||||||||
Extracting the migrations from a file/directory resource | (defn extract-file-migrations
[^File location-folder]
(let [folder-seq (file-seq location-folder)]
(->>
folder-seq
rest
(map #(validate-is-a-file %))
(map #(validate-file-can-be-read %))
(map #(generate-basic-structure-from-file %))))) | ||||||||||||||||||||||||
Create a map that contains all the properties from a Virtual File System | (defn generate-basic-structure-from-vfs
[^VirtualFile v]
{:file-name (.getName v)
:file v
:source :vfs}) | ||||||||||||||||||||||||
Extracting the migrations from a VFS resource | (defn- extract-vfs-migrations
[^String location-folder]
(->>
location-folder
.getPath
(. URLDecoder decode)
File.
.getAbsolutePath
(. VFS getChild)
.getChildren
(mapv generate-basic-structure-from-vfs))) | ||||||||||||||||||||||||
Create a map that contains all the properties from a JAR file | (defn- generate-basic-structure-from-jar
[migration-folder jar-entry]
{:file-name (subs (.getName jar-entry) (count migration-folder) (count (.getName jar-entry)))
:file (resource (.getName jar-entry))
:source :jar}) | ||||||||||||||||||||||||
Extracting the migrations from a JAR resource | (defn- extract-jar-migrations
[[^String jar-path ^String migration-folder]]
(->>
jar-path
JarFile.
.entries
enumeration-seq
(filter #(re-find (re-pattern (str migration-folder "*")) (.getName %)))
(filter #(re-find #"\.edn{1}|\.clj{1}" (.getName %)))
(mapv (partial generate-basic-structure-from-jar migration-folder)))) | ||||||||||||||||||||||||
It splits the jar path and the internal folder from the jar URL from jar:file:/Users/haduart/flycouchdb-example-0.1.0-SNAPSHOT-standalone.jar!/migrations/ to ['/Users/haduart/flycouchdb-example-0.1.0-SNAPSHOT-standalone.jar' 'migrations/'] | (defn- parse-jar-path-and-folder
[^URL jar-url]
(let [jar-file (str jar-url)
jar-string (subs jar-file (count "jar:file:") (count jar-file))
jar-position (.indexOf jar-string ".jar!")]
[(str (subs jar-string 0 jar-position) ".jar")
(subs jar-string (+ jar-position (count ".jar!/")) (count jar-string))])) | ||||||||||||||||||||||||
Wraps around the Java call. Just to make the function testable | (defn- get-url-protocol [location-folder] (.getProtocol location-folder)) | ||||||||||||||||||||||||
(defmulti extract-migrations
(fn [{location-folder :location-folder}] (get-url-protocol location-folder))) | |||||||||||||||||||||||||
(defmethod extract-migrations "jar" [{location-folder :location-folder}]
(->
location-folder
parse-jar-path-and-folder
extract-jar-migrations)) | |||||||||||||||||||||||||
(defmethod extract-migrations "file" [{location-folder :location-folder}]
(->
location-folder
file
extract-file-migrations)) | |||||||||||||||||||||||||
(defmethod extract-migrations "vfs" [{location-folder :location-folder}]
(->
location-folder
extract-vfs-migrations)) | |||||||||||||||||||||||||