commit c23e35d51673fa99a60d5dcaf615732f0f6aa784 Author: Marc Nijdam Date: Sun Dec 15 07:14:42 2019 -0400 First working version diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c9db396 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +.DS_Store +.env \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d68dea6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,23 @@ +### How to Contribute to this repository + +We value contributions from the community and will do everything we +can go get them reviewed in a timely fashion. If you have code to send +our way or a bug to report: + +* **Contributing Code**: If you have new code or a bug fix, fork this + repo, create a logically-named branch, and [submit a PR against this + repo](https://github.com/helium/blockchain-api). Include a + write up of the PR with details on what it does. + +* **Reporting Bugs**: Open an issue [against this + repo](https://github.com/helium/blockchain-api/issues) with as + much detail as you can. At the very least you'll include steps to + reproduce the problem. + +This project is intended to be a safe, welcoming space for +collaboration, and contributors are expected to adhere to the +[Contributor Covenant Code of +Conduct](http://contributor-covenant.org/). + +Above all, thank you for taking the time to be a part of the Helium +community. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2d4dfae --- /dev/null +++ b/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2018, Helium Systems Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2f16cdb --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +.PHONY: compile + +REBAR=./rebar3 + +compile: + $(REBAR) escriptize + +typecheck: + $(REBAR) do dialyzer,xref + +clean: + $(REBAR) clean diff --git a/README.md b/README.md new file mode 100644 index 0000000..14c6efd --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +psql_migration +===== + +An escript to help with Postgres migrations. + +This is inspired by the similarly named [pgsql_migration](https://github.com/artemeff/pgsql_migration) and the excellent tooling offered by [diesel](http://diesel.rs). + +Build +----- + + $ ./rebar3 escriptize + +Run +--- + +The script will look for a `DATABASE_URL` environment variable for connectivity details a running Postgres and suports `.env` file to hold that variable. + + +Run the script with: + + $ _build/default/bin/psql_migration + +See the help output for more details on options and commands. diff --git a/rebar.config b/rebar.config new file mode 100644 index 0000000..7401a75 --- /dev/null +++ b/rebar.config @@ -0,0 +1,23 @@ +{erl_opts, + [ + debug_info, + warnings_as_errors + ]}. + +{deps, + [ + {envloader, {git, "https://github.com/nuex/envloader.git", {branch, "master"}}}, + {eql, {git, "git://github.com/artemeff/eql.git", {tag, "0.1.0"}}}, + getopt, + epgsql + ]}. + +{escript_incl_apps, [psql_migration]}. +{escript_main_app, psql_migration}. +{escript_name, psql_migration}. +{escript_emu_args, "%%! +sbtu +A1\n"}. + +%% Profiles +{profiles, [{test, + [{erl_opts, [debug_info]} + ]}]}. diff --git a/rebar.lock b/rebar.lock new file mode 100644 index 0000000..10cadfc --- /dev/null +++ b/rebar.lock @@ -0,0 +1,16 @@ +{"1.1.0", +[{<<"envloader">>, + {git,"https://github.com/nuex/envloader.git", + {ref,"27a97e04f35c554995467b9236d8ae0188d468c7"}}, + 0}, + {<<"epgsql">>,{pkg,<<"epgsql">>,<<"4.3.0">>},0}, + {<<"eql">>, + {git,"git://github.com/artemeff/eql.git", + {ref,"a6727a3f878bfdd06648b7a886cbd2c0630db172"}}, + 0}, + {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0}]}. +[ +{pkg_hash,[ + {<<"epgsql">>, <<"26D9CF04D74773D1DC4DA24AD39E926B34E107232591FE1866EFDFBC0A098396">>}, + {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>}]} +]. diff --git a/rebar3 b/rebar3 new file mode 100755 index 0000000..00494f9 Binary files /dev/null and b/rebar3 differ diff --git a/src/psql_migration.app.src b/src/psql_migration.app.src new file mode 100644 index 0000000..da5bc85 --- /dev/null +++ b/src/psql_migration.app.src @@ -0,0 +1,18 @@ +{application, psql_migration, + [{description, "A script to manage Postgres migrations"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, + [kernel, + stdlib, + getopt, + epgsql, + eql, + envloader + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/src/psql_migration.erl b/src/psql_migration.erl new file mode 100644 index 0000000..9c21272 --- /dev/null +++ b/src/psql_migration.erl @@ -0,0 +1,210 @@ +-module(psql_migration). + +%% API exports +-export([main/1]). + +%%==================================================================== +%% API functions +%%==================================================================== + +-define(OPTS, + [ + {help, $h, "help", undefined, "Print this help text"}, + {dir, $d, "dir", {string, "migrations"}, "Migration folder"}, + {env, $e, "env", {string, ".env"}, "Environment file to search for DATABASE_URL"} + ]). + +%% escript Entry point +main(Args) -> + handle_command(getopt:parse(?OPTS, Args)). + +usage(ExitCode) -> + getopt:usage(?OPTS, "psql_migration", + "", + [ + {"new ", "Create a new migration"}, + {"list", "List migrations indicating which have been applied"}, + {"run", "Run all migrations"}, + {"revert", "Revert the last migration"} + ]), + halt(ExitCode). + +handle_command({error, Error}) -> + io:format("~p~n~n", [Error]), + usage(1); +handle_command({ok, {[help | _], _}}) -> + usage(0); +handle_command({ok, {Args, ["list"]}}) -> + Applied = applied_migrations(Args), + Available = available_migrations(Args), + IsApplied = fun(Mig) -> + case lists:member(Mig, Applied) of + true -> "X"; + _ -> " " + end + end, + io:format("Migrations:~n"), + [io:format(" [~s] ~s~n", [IsApplied(Entry), Entry]) || {Entry, _} <- Available]; +handle_command({ok, {Args, ["new", Name]}}) -> + Dir = target_dir(Args), + Timestamp = erlang:system_time(seconds), + MigrationName = [integer_to_list(Timestamp), "-", Name, ".sql"], + Filename = filename:join(Dir, MigrationName), + C = ["-- ", Filename, "\n", + "-- :up\n", + "-- Up migration\n\n", + "-- :down\n", + "-- Down migration\n" + ], + file:write_file(Filename, list_to_binary(C), [exclusive]), + io:format("Created migration: ~s~n", [Filename]); +handle_command({ok, {Args, ["run"]}}) -> + Available = available_migrations(Args), + case open_connection(Args) of + {ok, Conn} -> + Applied = applied_migrations(Conn), + ToApply = lists:filter(fun ({Mig, _}) -> not lists:member(Mig, Applied) end, Available), + Results = apply_migrations(up, ToApply, Conn), + report_migrations(up, Results); + {error, Error} -> + io:format("Failed to connect to database: ~p~n", [Error]), + halt(1) + end; +handle_command({ok, {Args, ["revert"]}}) -> + Available = available_migrations(Args), + case open_connection(Args) of + {ok, Conn} -> + case applied_migrations(Conn) of + [] -> + io:format("No applied migrations to revert"), + halt(1); + Applied -> + LastApplied = lists:last(Applied), + case lists:keyfind(LastApplied, 1, Available) of + false -> + io:format("Migration ~p can not be found~n", [LastApplied]), + halt(1); + Migration -> + Results = apply_migrations(down, [Migration], Conn), + report_migrations(down, Results) + end + end; + {error, Error} -> + io:format("Failed to connect to database: ~p~n", [Error]), + halt(1) + end; +handle_command({ok, {_, _}}) -> + usage(1). + + + +%% Utils + +open_connection(Args) -> + envloader:load(dot_env(Args)), + URL = os:getenv("DATABASE_URL"), + ParseOpts = [{scheme_defaults, [{postgres, 5432}]}], + case http_uri:parse(URL, ParseOpts) of + {error, Error} -> + {error, Error}; + {ok, {postgres, UserPass, Host, Port, Database, _}} -> + [User, Pass] = string:split(UserPass, ":"), + OpenOpts = #{ port => Port, + database => string:slice(Database, 1)}, + epgsql:connect(Host, User, Pass, OpenOpts); + {ok, {Schema, _, _, _, _, _}} -> + {error, {unsupported_schema, Schema}} + end. + + +target_dir(Args) -> + case lists:keyfind(dir, 1, Args) of + false -> "."; + {dir, Dir} -> + filelib:ensure_dir(Dir), + Dir + end. + +dot_env(Args) -> + case lists:keyfind(env, 1, Args) of + false -> ".env"; + {env, DotEnv} -> DotEnv + end. + +-spec report_migrations(up | down, [{Version::string(), ok | {error, term()}}]) -> ok. +report_migrations(_, L) when length(L) == 0 -> + io:format("No migrations were run~n"); +report_migrations(up, Results) -> + [io:format("Applied ~s: ~p~n", [V, R]) || {V, R} <- Results]; +report_migrations(down, Results) -> + [io:format("Reverted ~s: ~p~n", [V, R]) || {V, R} <- Results]. + +-define(DRIVER, epgsql). + +record_migration(up, Conn, V) -> + ?DRIVER:equery(Conn, "INSERT INTO __migrations (id) VALUES ($1)", [V]); +record_migration(down, Conn, V) -> + ?DRIVER:equery(Conn, "DELETE FROM __migrations WHERE id = $1", [V]). + +apply_migrations(Type, Migrations, Conn) -> + Results = lists:foldl(fun(_, [{_, {error, _}} | _]=Acc) -> + Acc; + (Migration={Version, _}, Acc) -> + case apply_migration(Type, Migration, Conn) of + ok -> [{Version, ok} | Acc]; + {error, Error} -> [{Version, {error, Error}}] + end + end, [], Migrations), + lists:reverse(Results). + +apply_migration(Type, {Version, Migration}, Conn) -> + Query = eql:get_query(Type, Migration), + ?DRIVER:squery(Conn, "BEGIN;"), + case if_ok(?DRIVER:squery(Conn, Query)) of + ok -> + record_migration(Type, Conn, Version), + ?DRIVER:squery(Conn, "COMMIT;"), + ok; + {error, Error} -> + ?DRIVER:squery(Conn, "ROLLBACK;"), + {error, Error} + end. + +if_ok(Rs) when is_list(Rs) -> + Result = lists:map(fun(R) -> if_ok(R) end, Rs), + case lists:keyfind(error, 1, Result) of + false -> ok; + Error -> Error + end; +if_ok({ok, _}) -> ok; +if_ok({ok, _, _}) -> ok; +if_ok({ok, _, _, _}) -> ok; +if_ok({error, {error,error,_,_,Descr,_}}) -> {error, Descr}; +if_ok(Error) -> {error, Error}. + +available_migrations(Args) -> + Dir = target_dir(Args), + {ok, Files} = file:list_dir(Dir), + lists:map(fun(Filename) -> + {ok, Migs} = eql:compile(filename:join([Dir, Filename])), + {filename:rootname(Filename), Migs} + end, lists:usort(Files)). + +applied_migrations(Args) when is_list(Args) -> + case open_connection(Args) of + {ok, Conn} -> + applied_migrations(Conn); + {error, Error} -> + {error, Error} + end; +applied_migrations(Conn) -> + case ?DRIVER:squery(Conn, "SELECT id FROM __migrations ORDER by id ASC") of + {ok, _, Migs} -> [binary_to_list(Mig) || {Mig} <- Migs]; + {error, {error, error, <<"42P01">>, _, _, _}} -> + %% init migrations and restart + {ok, _, _} = ?DRIVER:squery(Conn, + "CREATE TABLE __migrations (" + "id VARCHAR(255) PRIMARY KEY," + "datetime TIMESTAMP DEFAULT CURRENT_TIMESTAMP)"), + applied_migrations(Conn) + end.