A Lightweight OCaml Webapp Tutorial

3.2.1 Install and initialize opam

sh <(curl -sL https://raw.githubusercontent.com/ocaml/opam/master/shell/install.sh)

opam init
# Install any dependencies called for
# Answer yes to the prompts to configure your shell environment

opam switch 4.08.1
# Sets the system ocaml compiler to use ocaml 4.08.1
# Use a different version at your own discretion

3.2.2 Install dune

opam install dune

3.2.3 Install editor support and initialize your configurations

opam install user-setup merlin ocamlformat ocp-indent
opam user-setup install

1. Emacs
   

   Install the OCaml mode for emacs

   opam install tuareg
   

   If you aren’t already an emacs user but want to give it a spin, I highly recommend the spacemacs distribution for a beginner friendly approach.

   I use doom-emacs, and find the IDE experience phenomenal.

2. For Vim or VSCode
   

   See recommendations on the OCamlveres.

3.2.4 Install PostgreSQL

3.3.1 Project initialization

dune init proj ocaml_webapp \
    --libs core,opium,caqti,caqti-driver-postgresql,caqti-lwt,tyxml,lwt.unix \
    --ppx ppx_rapper

3.3.2 Configuration of the package and its dependencies

(lang dune 2.0)
(generate_opam_files true)

(name ocaml_webapp)
(version 0.1.0)

(authors "Shon Feder")
(license MIT) ; TODO
(source (uri "git+https://gitlab.com/shonfeder/ocaml_webapp.git")) ; TODO
(maintainers "shon.feder@gmail.com") ; TODO
(homepage "https://gitlab.com/shonfeder/ocaml_webapp") ; TODO
(bug_reports "https://gitlab.com/shonfeder/ocaml_webapp/issues") ; TODO
(documentation "https://shonfeder.gitlab.io/ocaml_webapp/") ; TODO

(package
 (name ocaml_webapp)
 (synopsis "A minimal example of a lightweight webapp in OCaml") ; TODO
 (description "A minimal webapp using Opium, Catqi, and tyXML") ; TODO
 (depends
  ;; General system dependencies
  (dune (>= 2))
  (ocaml (>= 4.08.0))

  ;; Standard library replacement
  (core (>= v0.12.2))

  ;; Web toolkit
  (opium (>= 0.17.1))

  ;; Database interface
  (caqti (>= 1.2.3))
  (caqti-lwt (>= 1.2.0))
  (caqti-driver-postgresql (>= 1.2.4))
  (ppx_rapper (>= 2.0.0))

  ;; HTML generation
  (tyxml (>= 4.3.0))

  ;; Logging
  (logs (>= 0.7.0))

  ;; Dev dependencies
  (utop :dev)
  (merlin :dev)
  (ocamlformat :dev)
  (ocp-indent :dev)
  (tuareg :dev) ;; rm if not using emacs
))

3.3.3 The Makefile

# The name of your project (as given in the dune-project file)
# TODO
project_name = ocaml_webapp

# The opam package configuration file
opam_file = $(project_name).opam

.PHONY: deps run run-debug migrate rollback

# Alis to update the opam file and install the needed deps
deps: $(opam_file)

# Build and run the app
run:
    dune exec $(project_name)

# Build and run the app with Opium's internal debug messages visible
run-debug:
    dune exec $(project_name) -- --debug

# Run the database migrations defined in migrate/migrate.ml
migrate:
    dune exec migrate_ocaml_webapp

# Run the database rollback defined in migrate/rollback.ml
rollback:
    dune exec rollback_ocaml_webapp

# Update the package dependencies when new deps are added to dune-project
$(opam_file): dune-project
    -dune build @install        # Update the $(project_name).opam file
    -git add $(opam_file)       # opam uses the state of master for it updates
    -git commit $(opam_file) -m "Updating package dependencies"
    opam install . --deps-only  # Install the new dependencies

3.4.1 A fresh opam switch for the project

# From within the project root directory
opam switch create . 4.08.1 --deps-only
# I recommend 4.08.1 because all basic tools are compatible with it as of 2019/12/30
# if this should need updating, please make an issue or PR

opam switch
# →/home/you/path/to/ocaml_webapp  ocaml-base-compiler.4.08.1  /home/you/path/to/ocaml_webapp

3.4.2 Install dune in the switch

opam install dune

3.4.3 Install and update the project dependencies

make deps
# install all dependencies

3.4.4 Confirm that the app skeleton can build

dune build

3.5.1 Troubleshooting problems with the database port

11: (** Configuration of the connection *)
12: let url = "localhost"
13: let port = 5432
14: let database = "ocaml_webapp"

4.1.1 Explanation of the code

1. Execution
   

   The first thing we do is set up logging with the set_logger function³. Doing this first ensures that any important log messages that may be generated during subsequent actions will be recorded.

   App.run_command' app generates a commandline interface for launching our application app. It returns a value of type unit Lwt.t, signifying the deferred result of running the program.

   You can read the docs for the generated command line interface by running

   dune exec ocaml_webapp -- --help
   

   We pass the deferred result to Lwt_main.run, which kicks of the Lwt scheduler, and runs our app (see the Lwt_main docs for more info).

2. Setting the logger
   

   The set_logger function itself merely uses the default logs log reporter. It sets the log_level, which we’ve hard coded to Logs.Debug. We use the Lwt binding operator to sequence our actions here, just as we do in the run function.

   Note that for this logging configuration to work, we rely on the library dependencies logs, logs.fmt, and logs.lwt declared in ./bin/dune:

   Listing 5: Declaration of library dependencies from the logs package in ./bin/dune

    9:    logs         ;; The logs library
   10:    logs.fmt     ;; The logs format helper
   11:    logs.lwt     ;; The logs concurrency helper
   

   To see the debug logging output (very useful for tracking down routing problems) run the app with

   make run-debug
   

   This will output all the incoming and outgoing HTTP requests to your terminal.

   NOTE: The --debug flag used in the invocation of Opium here only affects the internal logging from the Opium library. It is the log_level we set that effects the output level from the logger.

3. Building the app
   

   let app = ... defines our application. Applications are built up by pushing a base App.empty through a pipeline of functions of type App.builder. App.builder is just a synonym for functions of type App.t -> App.t, which take in an application, add something to it, and pass along the new application. In this case, we build the app with the follow steps:

   1. Set the name for the command line interface with App.cmd_name
   2. Add the middleware to serve static files
   3. Add the database connection to the environment with the Db.middleware function
   4. Add all of the route handlers with Route.add_routes.

   The helper static just configures a piece of middleware to serve static files, using Opium’s Middleware.static. It serves the contents of the ./static directory at the "/static" route.

4.2.1 Explanation of the code

1. Responding to requests
   

   Each function is defined in terms of a route combinator that takes a string encoding the route and a handler function. The handler function takes an incoming request (req) to an outgoing response.

   The root and hello_fallback handlers don’t use the request. Instead, they use respond’ to reply with the relevant page defined in the Content module. The ' on the respond' function indicates that the respone is a deferred Lwt.t value. We’ll use this for all our respones, and Lwt will handle all the concurrency for us.

   Note that the hello_fallback handler offers the first appearance of HTML generation: Html.[p [txt "Hiya"]] invokes the p and txt functions from the Tyxml’s Html module to generate a paragraph HTML element holding the text "Hiya". We will explore this deeper in the section on Content Generation.

2. Handling path parameters
   

   The hello function handles routes that include a segment after the "hello". The :lang component of the path will cause a "lang" parameter to be added to the req. This parameter will hold a string recording the value of this segment of the path for any incoming request that matches the pattern. "/hello/Hindi" matches this pattern but "/hello/foo/bar" and "/hello" do not.

   Here, we just pass the lang parameter on to our page generation function Content.hello_page.

4.3.1 Basics of Tyxml as Seen in the <head> Element

 1: open Core
 2: open Tyxml
 3: 
 4: (** A <head> component shared by all pages *)
 5: let default_head =
 6:   let open Html in
 7:   head
 8:     (title (txt "OCaml Webapp Tutorial"))
 9:     [ meta ~a:[a_charset "UTF-8"] ()
10:     ; link ~rel:[`Stylesheet] ~href:"/static/style.css" () ]

1. Explanation of the code
   

   Tyxml.Html aims to provide type-safe combinators for every valid HTML element. The name of the combinator is generally the same as the tag for the element itself. Most combinators take an optional named ~a argument that accepts a list of attributes, and a required argument with a list of values of type Tyxml.Html.elt which specify the element’s inner HTML.

   In the default_head function, we see head, which generates <head>...</head> elements. It is an exception to the general pattern of element combinators: since every well formed head element should have a title, head takes a required title argument, and then the usual list of other elements. The title element itself takes a terminal txt expression.

   meta takes a () instead of a list of elements. This is because it is a terminal element. The meta element shows an example of setting attributes: Txyml.Html attributes combinators all begin with the a_ prefix. In this case, we declare the charset attribute for our HTML documents.

   The above code generates the following HTML:

   <head>
     <title>OCaml Webapp Tutorial</title>
     <meta charset="UTF-8"/>
     <link rel="stylesheet" href="/static/style.css"/>
   </head>
   

4.3.2 Generating HTML for Use by Opium

11: (** The basic page layout, emitted as an [`Html string] which Opium can use as a
12:     response *)
13: let basic_page content =
14:   let raw_html =
15:     let open Html in
16:     html default_head (body content)
17:     |> Format.asprintf "%a" (Html.pp ())
18:   in
19:   `Html raw_html

1. Explanation of the code
   

   This function is responsible for the following preparation of its content:

   1. Wrap the content in a <body> tag
   2. Construct an html element with the default_head
   3. Serialize the AST of type _ Tyxml.Html.elt into a well formatted string
   4. Wrap the raw_html in the `Html polymorphic variant tag expected by opium

   All the HTML we construct for our app will ultimately go trough basic_page prior to becoming part of an opium response.

4.3.3 Parametric Templates

14: (** Defines a handler that takes a path parameter from the route *)
15: let hello = get "/hello/:lang"
16:     begin fun req ->
17:       let lang = param req "lang" in
18:       respond' @@
19:       Content.hello_page lang
20:     end

41: let hello_page lang =
42:   let greeting = match lang with
43:     | "中文"    -> "你好，世界!"
44:     | "Deutsch" -> "Hallo, Welt!"
45:     | "English" -> "Hello, World!"
46:     | _         -> "Language not supported :(\nYou can add a language via PR to https://gitlab.com/shonfeder/ocaml_webapp"
47:   in
48:   basic_page Html.[p [txt greeting]]

4.3.4 Forms that Submit Post Requests

50: let add_excerpt_page =
51:   let txt_input name =
52:     Html.[ label ~a:[a_label_for name] [txt (String.capitalize name)]
53:          ; input ~a:[a_input_type `Text; a_name name] () ]
54:   in
55:   let excerpt_input =
56:     let name = "excerpt" in
57:     Html.[ label ~a:[a_label_for name] [txt (String.capitalize name)]
58:          ; textarea ~a:[a_name name] (txt "") ]
59:   in
60:   let submit =
61:     Html.[input ~a:[ a_input_type `Submit; a_value "Submit"] () ]
62:   in
63:   basic_page
64:     Html.[ form ~a:[a_method `Post; a_action "/excerpts/add"]
65:              (List.map ~f:p
66:                 [ txt_input "author"
67:                 ; excerpt_input
68:                 ; txt_input "source"
69:                 ; txt_input "page"
70:                 ; submit
71:                 ])]

29: let get_excerpts_add = get "/excerpts/add" begin fun _req ->
30:     respond' @@
31:     Content.add_excerpt_page
32:   end

4.4.1 Implementing the Database Connection

11: (** Configuration of the connection *)
12: let url = "localhost"
13: let port = 5432
14: let database = "ocaml_webapp"
15: let connection_uri = Printf.sprintf "postgresql://%s:%i/%s" url port database
16: 
17: (* [connection ()] establishes a live database connection and is a pool of
18:    concurrent threads for accessing that connection. *)
19: let connect () =
20:   connection_uri
21:   |> Uri.of_string
22:   |> Caqti_lwt.connect_pool ~max_size:10
23:   |> function | Ok pool   -> pool
24:               | Error err -> failwith (Caqti_error.show err)
25: 
26: (* [query_pool query pool] is the [Ok res] of the [res] obtained by executing
27:    the database [query], or else the [Error err] reporting the error causing
28:    the query to fail. *)
29: let query_pool query pool =
30:   Caqti_lwt.Pool.use query pool

1. Explanation of the code
   

   The purpose of the connect function is largely to seal in the gnarly, polymorphic variant error type used by Caqti, so that the rest of the app doesn’t need to be aware of it. We map the error type to a string representation of it, but a productionized app you’d likely want to create a more robust error handler. In a production app, we’d want more robust error handling than just throwing failwith (probably using a result type). The same goes for the err handling in the query_pool helper:

   NOTE: The configuration variables hard code our database connection parameters into the source, which is expedient for this tutorial, but not a good practice for a real project, where we’d want to take these out of a configuration file or environment variable.

4.4.2 Implementing the database middleware

40: (* Seal the key type with a non-exported type, so the pool cannot be retrieved
41:    outside of this module *)
42: type 'err db_pool = 'err caqti_conn_pool
43: let key : _ db_pool Opium.Hmap.key = Opium.Hmap.Key.create ("db pool" , fun _ -> sexp_of_string "db_pool")
44: 
45: (* Initiate a connection pool and add it to the app environment *)
46: let middleware app =
47:   let pool = connect ()
48:   in
49:   let filter handler (req : Request.t) =
50:     let env = Opium.Hmap.add key pool (Request.env req) in
51:     handler {req with env}
52:   in
53:   let m = Rock.Middleware.create ~name:"database connection pool" ~filter in
54:   middleware m app
55: 
56: (* Execute a query on the database connection pool stored in the request
57:    environment *)
58: let query_db query req =
59:   Request.env req
60:   |> Opium.Hmap.get key
61:   |> query_pool query

1. Explanation of the code
   

   The key is used to look up the database connection from the environment (the environment being a heterogeneous valued hashmap implemented using Hmap). The db_pool type alias is kept private, and we do not export the key itself, so there is no way to retrieve the connection from the environment outside of the Db module.

   A piece of middleware is of type Opium.App.builder, i.e., Opium.App.t -> Optium.App.t. All of the Db middleware logic is in its inline filter function. In general, such filters take a handler and a request, req, do some stuff to construct a new request, and then handle the new request with the given handler. In this case, all we’re doing is making sure that the database connection pool is available in the environment of the request. We use opium’s Rock.Middleware.create to create our new middleware and apply it to the given app using Opium.Std.middleware.

   The query_db function is just a helper used to execute queries on the database connection stored in a request. If we wanted to let the client code execute arbitrary queries on the database connection, we could expose this function in the API. However, we choose to keep it hidden, so that the database can only be accessed through the API we define below in the Get and Update submodules.

4.4.3 Writing Database Queries using ppx_rapper

 56: (** Collects all the SQL queries *)
 57: module Query = struct
 58:   type ('res, 'err) query_result = ('res, [> Caqti_error.call_or_retrieve ] as 'err) result Lwt.t
 59: 
 60:   let add_excerpt
 61:     : Excerpt.t -> Caqti_lwt.connection -> (unit, 'err) query_result =
 62:     let open Excerpt in
 63:     [%rapper execute
 64:         {sql|
 65:         INSERT INTO excerpts(author, excerpt, source, page)
 66:         VALUES (%string{author}, %string{excerpt}, %string{source}, %string?{page})
 67:         |sql}
 68:         record_in
 69:     ]
 70: 
 71:   let get_excerpts_by_author
 72:     : author:string -> Caqti_lwt.connection -> (Excerpt.t list, 'err) query_result =
 73:     let open Excerpt in
 74:     [%rapper get_many
 75:         {sql|
 76:         SELECT @string{author}, @string{excerpt}, @string{source}, @string?{page}
 77:         FROM excerpts
 78:         WHERE author = %string{author}
 79:         |sql}
 80:         record_out
 81:     ]
 82: 
 83:   let get_authors
 84:     : unit -> Caqti_lwt.connection -> (string list, 'err) query_result =
 85:     [%rapper get_many
 86:         {sql|
 87:         SELECT DISTINCT @string{author}
 88:         FROM excerpts
 89:         |sql}
 90:     ]
 91: end
 92: 
 93: (* Execute queries for fetching data *)
 94: module Get = struct
 95:   let excerpts_by_author author : Request.t -> (Excerpt.t list, string) Lwt_result.t =
 96:     query_db (Query.get_excerpts_by_author ~author)
 97: 
 98:   let authors : Request.t -> (string list, string) Lwt_result.t =
 99:     query_db (fun c -> Query.get_authors () c)
100: end
101: 
102: (* Execute queries for updating data *)
103: module Update = struct
104:   let add_excerpt excerpt : Request.t -> (unit, string) Lwt_result.t =
105:     query_db (fun c -> Query.add_excerpt excerpt c)
106: end

1. Explanation of the code
   

   Query.add_excerpt will execute the query defined in the sql string. The function generated by ppx_rapper has no output values, as indicated by the absence of any leading @, so it will return a unit. The SQL template takes four input values, %string{author}, %string{excerpt}, %string{source}, and string?{page}. %string?{page} indicates that page will be of type string option. The use of the record_in dictates that these values will be taken from a record with the corresponding field names and types; i.e., a record of type Excerpt.t:

   Listing 18: The type for Excerpt.t records, defined in ./lib/excerpt.ml

   1: type t =
   2:   { author: string
   3:   ; excerpt: string
   4:   ; source: string
   5:   ; page: string option
   6:   }
   

   The resulting type of Query.add_excerpt expresses that the function takes a connection to the database and inserts the excerpt into it, returning an Ok () if all goes well.

   In the query function get_excerpts_by_author we see a query which will get_many results. Each result will be a record_out, with fields and types as dictated by the terms prefixed with a @, i.e., the Excerpt.t record. We also see that the query takes one input parameter, the %string{author}. Since we do not specify that we’re expecting a record_in, the resulting function will take a named parameter ~author:string.

   These query functions are used in the Get and Update submodules exposed in the external API. Each function in the external API ends up taking a Request.t from the Opium App.t and returning any fetched values as an Lwt.t (Ok res) (or an error).

4.4.4 Using the Db API in a Route

34: let respond_or_err resp = function
35:   | Ok v      -> respond' @@ resp v
36:   | Error err -> respond' @@ Content.error_page err
37: 
38: let excerpt_of_form_data data =
39:   let find data key =
40:     let open Core in
41:     (* NOTE Should handle error in case of missing fields *)
42:     List.Assoc.find_exn ~equal:String.equal data key |> String.concat
43:   in
44:   let author  = find data "author"
45:   and excerpt = find data "excerpt"
46:   and source  = find data "source"
47:   and page    = match find data "page" with "" -> None | p -> Some p
48:   in
49:   Lwt.return Excerpt.{author; excerpt; source; page}
50: 
51: let post_excerpts_add = post "/excerpts/add" begin fun req ->
52:     let open Lwt in
53:     (* NOTE Should handle possible error arising from invalid data *)
54:     App.urlencoded_pairs_of_body req  >>=
55:     excerpt_of_form_data              >>= fun excerpt ->
56:     Db.Update.add_excerpt excerpt req >>=
57:     respond_or_err (fun () -> Content.excerpt_added_page excerpt)
58:   end

1. Explanation of the code
   

   The respond_or_err helper deals with possible Error err values of a result, passing Ok values back as responses if possible, or else responding with an error page.

   The excerpt_of_form_data function takes an association list of key-value pairs representing form data and constructs an Excerpt.t record.

   Finally, the post_excerpts_add handler decodes a response body containing form data (encoded according to the application/x-www-form-urlencoded content type) into an association list, uses the previous helper to build an Excerpt.t, and then gives the Excerpt.t to the Db.Update.add_excerpt function, which inserts the Excerpt.t into the database via the connection stored in the environment of the req.

4.4.5 Implementation of a Rudimentary Database Migration Utility

108: module Migration = struct
109:   type 'a migration_error =
110:     [< Caqti_error.t > `Connect_failed `Connect_rejected `Post_connect ] as 'a
111: 
112:   type 'a migration_operation =
113:     unit -> Caqti_lwt.connection -> (unit, 'a migration_error) result Lwt.t
114: 
115:   type 'a migration_step = string * 'a migration_operation
116: 
117:   let execute migrations =
118:     let open Lwt in
119:     let rec run migrations pool = match migrations with
120:       | [] -> Lwt_result.return ()
121:       | (name, migration) :: migrations ->
122:         Lwt_io.printf "Running: %s\n" name        >>= fun () ->
123:         query_pool (fun c -> migration () c) pool >>= function
124:         | Ok () -> run migrations pool
125:         | Error err -> return (Error err)
126:     in
127:     return (connect ()) >>= run migrations
128: end