Beginnings of making frontend pretty, switched to pico css

running commandsserver side
* More robust streaming responses client side

.ameba.yml

@@ -1,18 +1,17 @@
 # This configuration file was generated by `ameba --gen-config`
-# on 2024-07-02 16:33:24 UTC using Ameba version 1.6.1.
+# on 2024-07-07 14:44:11 UTC using Ameba version 1.6.1.
 # The point is for the user to remove these configuration records
 # one by one as the reported problems are removed from the code base.
 
-# Problems found: 11
+# Problems found: 5
 # Run `ameba --only Documentation/DocumentationAdmonition` for details
 Documentation/DocumentationAdmonition:
   Description: Reports documentation admonitions
   Timezone: UTC
   Excluded:
-  - src/faaso.cr
-  - src/daemon/main.cr
+  - src/secrets.cr
   - src/daemon/secrets.cr
-  - src/funko.cr
+  - src/daemon/funko.cr
   - spec/faaso_spec.cr
   Admonitions:
   - TODO
@@ -21,6 +20,16 @@ Documentation/DocumentationAdmonition:
   Enabled: true
   Severity: Warning
 
+# Problems found: 1
+# Run `ameba --only Lint/UselessAssign` for details
+Lint/UselessAssign:
+  Description: Disallows useless variable assignments
+  ExcludeTypeDeclarations: false
+  Excluded:
+  - src/daemon/config.cr
+  Enabled: true
+  Severity: Warning
+
 # Problems found: 3
 # Run `ameba --only Naming/BlockParameterName` for details
 Naming/BlockParameterName:
@@ -28,8 +37,8 @@ Naming/BlockParameterName:
   MinNameLength: 3
   AllowNamesEndingInNumbers: true
   Excluded:
-  - src/faaso.cr
-  - src/daemon/funkos.cr
+  - src/daemon/funko.cr
+  - src/commands/build.cr
   AllowedNames:
   - _
   - e

.gitignore

@@ -5,3 +5,6 @@
 tmp/
 export/
 secrets/
+.rucksack
+.rucksack.toc
+.faaso.yml

.hadolint.yml

@@ -0,0 +1,3 @@
+ignored:
+  - DL3018
+  - DL3059

.pre-commit-config.yaml

@@ -0,0 +1,38 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+repos:
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v4.4.0
+      hooks:
+          - id: trailing-whitespace
+          - id: end-of-file-fixer
+          - id: check-yaml
+          - id: check-added-large-files
+          - id: check-merge-conflict
+    - repo: https://github.com/jumanjihouse/pre-commit-hooks
+      rev: 3.0.0
+      hooks:
+          - id: shellcheck
+          - id: markdownlint
+            exclude: '^content'
+    - repo: https://github.com/mrtazz/checkmake
+      rev: 0.2.2
+      hooks:
+          - id: checkmake
+    - repo: https://github.com/python-jsonschema/check-jsonschema
+      rev: 0.23.2
+      hooks:
+          - id: check-github-workflows
+    - repo: local
+      hooks:
+        - id: empty-funkos
+          name: empty-funkos
+          entry: test ! -s config/funkos
+          language: system
+          pass_filenames: false
+    - repo: https://github.com/hadolint/hadolint
+      rev: v2.12.0
+      hooks:
+        - id: hadolint-docker
+          exclude: 'j2$'

Dockerfile

@@ -1,28 +1,48 @@
-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine as build
-RUN apk add crystal shards yaml-dev openssl-dev zlib-dev libxml2-dev
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine:3.20 AS build
+RUN apk add --no-cache \
+    crystal \
+    shards \
+    yaml-dev \
+    openssl-dev \
+    zlib-dev \
+    libxml2-dev \
+    make
+RUN rm -rf /var/cache/apk/*
 RUN addgroup -S app && adduser app -S -G app
 WORKDIR /home/app
-COPY shard.yml ./
+COPY shard.yml Makefile ./
 RUN mkdir src/
 COPY src/ src/
-RUN shards install
-RUN shards build -d --error-trace
-RUN strip bin/*
+COPY runtimes/ runtimes/
+RUN make
+# RUN strip bin/*
 
-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine as ship
-RUN apk add tinyproxy multirun openssl zlib yaml pcre2 gc libevent libgcc libxml2
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine:3.20 AS ship
+RUN apk add --no-cache \
+    caddy \
+    nss-tools \
+    multirun \
+    docker \
+    openssl \
+    zlib \
+    yaml \
+    pcre2 \
+    gc \
+    libevent \
+    libgcc \
+    libxml2 \
+    ttyd
+RUN rm -rf /var/cache/apk/*
 
 # Unprivileged user
 RUN addgroup -S app && adduser app -S -G app
 WORKDIR /home/app
+RUN mkdir /home/app/tmp && chown app /home/app/tmp
 
-RUN mkdir runtimes public
-COPY runtimes/ runtimes/
 COPY public/ public/
-COPY tinyproxy.conf ./
 COPY --from=build /home/app/bin/faaso-daemon /home/app/bin/faaso /usr/bin/
 
-RUN mkdir /secrets
-RUN echo "sarasa" > /secrets/sarlanga
+# Mount points for persistent data
+RUN mkdir /secrets /config
 
-CMD ["/usr/bin/multirun", "-v", "faaso-daemon", "tinyproxy -d -c tinyproxy.conf"]
+CMD ["/usr/bin/multirun", "-v", "faaso-daemon", "caddy run --config config/Caddyfile"]

Makefile

@@ -1,9 +1,26 @@
-build: shard.yml $(wildcard src/**/*cr)
-	shards build
-proxy: build
-	docker build . -t faaso-proxy --no-cache
+build: shard.yml $(wildcard src/**/*) $(runtimes/**/*)
+	shards build -d --error-trace
+	cat .rucksack >> bin/faaso
+	cat .rucksack >> bin/faaso-daemon
+proxy:
+	docker build . -t faaso-proxy
+
+all: build proxy
+
 start-proxy:
-	docker run --name faaso_proxy --rm --network=faaso-net -v /var/run/docker.sock:/var/run/docker.sock -v secrets:/home/app/secrets -p 8888:8888 faaso-proxy
+	docker network create faaso-net || true
+	docker run --name faaso-proxy-one \
+	--rm --network=faaso-net \
+	-e FAASO_SECRET_PATH=${PWD}/secrets \
+	-v /var/run/docker.sock:/var/run/docker.sock \
+	-v ${PWD}/secrets:/home/app/secrets \
+	-v ${PWD}/config:/home/app/config \
+	-p 8888:8888 faaso-proxy
 
+test:
+	crystal spec
 
-.PHONY: build proxy-image start-proxy
+clean:
+	rm bin/*
+
+.PHONY: all build proxy-image start-proxy test clean

README.md

@@ -50,22 +50,25 @@ This will give you:
 You need a server, with docker. In that server, build it as explained above.
 You can run the `faaso-proxy` with something like this:
 
-```
-docker run --network=faaso-net -v /var/run/docker.sock:/var/run/docker.sock -p 8888:8888 faaso-proxy
+```shell
+docker run --network=faaso-net \
+-v /var/run/docker.sock:/var/run/docker.sock \
+-p 8888:8888 faaso-proxy
 ```
 
-That will give `faaso-proxy` access to your docker, and expose the functionality in
-port 8888.
+That will give `faaso-proxy` access to your docker, and expose the functionality
+in port 8888.
 
 ## What it Does
 
 ### Funkos
 
-In FaaSO you (the user) can create Funkos. Funkos are the moral equivalent of AWS 
+In FaaSO you (the user) can create Funkos. Funkos are the moral equivalent of AWS
 lambdas and whatever they are called in other systems. In short, they are simple
 programs that handle web requests.
 
-For example, here is a `hello world` level funko written using Crystal, a file called `funko.cr`:
+For example, here is a `hello world` level funko written using Crystal,
+a file called `funko.cr`:
 
 ```crystal
 get "/" do
@@ -82,40 +85,45 @@ runtime: crystal
 ```
 
 If you have those two files in a folder, that folder is a funko, which is called
-`hello` and FaaSO knows it's written in Crystal. In fact, it knows (because the crystal runtime explains that, don't worry about it yet) that it's part of an
+`hello` and FaaSO knows it's written in Crystal. In fact, it knows (because the
+crystal runtime explains that, don't worry about it yet) that it's part of an
 application written in the [Kemal framework](https://kemalcr.com/) and it knows
 how to create a whole container which runs the app, and how to check its health,
 and so on.
 
 But the funko has *the interesting bits* of the app.
 
-The full details of how to write funkos are still in flux, so not documenting 
-it for now. Eventually, you will be able to just write the parts you 
+The full details of how to write funkos are still in flux, so not documenting
+it for now. Eventually, you will be able to just write the parts you
 need to write to create funkos in different languages. It's easy!
 
-### So what can a funko do?
+### So what can a funko do
 
 Once you have a funko, you can *build* it, which will give you a docker image.
 
 ```faaso build --local myfunko/```
 
-Or you can export it and get rid of all the mistery of how your funko **really** works:
+Or you can export it and get rid of all the mistery of how your funko
+**really** works:
 
-```faaso export myfunko```
+```faaso export myfunko/ myfuko-exported```
 
-Or, once you built it, you can run it, and you will be able to see it using 
+Or, once you built it, you can run it, and you will be able to see it using
 `docker ps`:
 
-```faaso up myfunko```
+```faaso scale myfunko 1```
+
+Here `scale myfunko 1` simply means "run one instance of myfunko". You can run more
+than one, although currently only one is used by the proxy.
 
 ### The FaaSO proxy
 
 The proxy has a few goals:
 
 1) You can connect to it using `faaso` and have it build/run/etc your funkos.
-
    * This builds the funko in your machine: `faaso build -l myfunko/`
-   * This builds the funko in the server pointed at by FAASO_SERVER: `faaso build myfunko/`
+   * This builds the funko in the server pointed at by FAASO_SERVER:
+     `faaso build myfunko/`
 
    Yes, they are exactly the same thing. In fact, if you don't use the `-l` flag,
    faaso just tells the proxy "hey proxy, run *your* copy of faaso over there and
@@ -124,10 +132,10 @@ The proxy has a few goals:
 2) It automatically reverse-proxies to all funkos.
 
    If you deployed a funko called `hello` and your faaso proxy is at
-   `http://myserver:8888` then the `/` path in your funko is at 
+   `http://myserver:8888` then the `/` path in your funko is at
    `http://myserver:8888/funko/hello/`
 
-   This proxying is automatic, you don't need to do anything. As long as you 
+   This proxying is automatic, you don't need to do anything. As long as you
    build the image for your funko in the server and then start the funko in the
    server? It should work.
 
@@ -142,4 +150,4 @@ beyond bugfixes, since I am redesigning things all the time.
 
 ## Contributors
 
-- [Roberto Alsina](https://github.com/ralsina) - creator and maintainer
+* [Roberto Alsina](https://github.com/ralsina) - creator and maintainer

TODO.md

@@ -0,0 +1,43 @@
+# TODO LIST
+
+## Things that need doing before first release
+
+* User flow for initial proxy setup
+  * ✅ Setting up password
+  * Setting up hostname for Caddy's automatic HTTPS
+* Config UI in frontend?
+* Polish frontend UI **A LOT**
+* ✅ Version checks for consistency between client/server
+* ✅ Have 3 runtimes:
+  * ✅ Crystal + Kemal
+  * ✅ Python + Flask
+  * ✅ Nodejs + Express
+* Create a site
+  * Document
+    * FaaSO for app developers
+    * FaaSO for runtime developers
+    * FaaSO server setup
+    * APIs
+* Sanitize all inputs
+* ✅ Streaming responses in slow operations like scaling down
+  or building
+* ✅ Make more things configurable / remove hardcoded stuff
+  * ✅ Make server take options from file
+  * ✅ Make server take options from environment
+  * ✅ Make server password configurable
+  * ✅ admin/admin auth client side
+  * ✅ `faaso login` is not working properly yet with proxy
+* CD for binaries and images for at least arm64/x86
+* Multi-container docker logs [faaso logs -f FUNKO]
+* ✅ Configurable verbosity, support stderr/stdout split
+* ✅ Fix proxy reload / Make it reload on file changes
+* Implement `faaso help command`
+* ✅ Fix `export examples/hello_crystal` it has a `template/`
+* ✅ Implement zero-downtime rollout (`faaso deploy`)
+* ✅ Cleanup `tmp/whatever` after use
+* ✅ `faaso scale` remote is broken
+* ✅ Setup linters/pre-commit/etc
+
+## Things to do but not before release
+
+* Propagate errors from `run_faaso` to the remote client

config/Caddyfile

@@ -0,0 +1,22 @@
+{
+	http_port 8888
+	https_port 8887
+	local_certs
+}
+
+http://*:8888 {
+	forward_auth /admin/* http://127.0.0.1:3000 {
+		uri /auth
+		copy_headers {
+			Authorization
+		}
+	}
+
+	handle_path /admin/terminal/* {
+		reverse_proxy /* http://127.0.0.1:7681
+	}
+	handle_path /admin/* {
+		reverse_proxy /* http://127.0.0.1:3000
+	}
+      import funkos
+}

config/faaso.yml

@@ -0,0 +1 @@
+password: adminfoo

docs/DESIGN.md

@@ -38,7 +38,7 @@ up/downscaling, no multiple versions routed by header.
 Specifically: no downscaling to zero. It makes everything MUCH
 more complicated.
 
-# Function structure
+## Function structure
 
 Example using crystal, but it could be anything. Any function has
 an associated runtime, for example "crystal" or "python".
@@ -69,7 +69,9 @@ Probably some `metadata.yml` that is *not* in the template.
 * Files that should be copied along the function
 * Whatever
 
-# Implementation Ideas
+## Implementation Ideas
 
 * caddy for proxy? It's simple, fast, API-configurable.
-* Local docker registry for images? See https://www.docker.com/blog/how-to-use-your-own-registry-2/ (maybe later)
+* Local docker registry for images? See
+  [use own registry](https://www.docker.com/blog/how-to-use-your-own-registry-2/)
+  (maybe later)

docs/deployments.md

@@ -0,0 +1,45 @@
+# Braindump on deployments
+
+Problem: We have a funko, we want it deployed.
+
+## Variant 1: Local deployment
+
+Solution: Just start the funko's container locally. Done. It's mostly implemented.
+
+## Variant 2: Deploy to the server
+
+If the server doesn't have the image, we have "server build", so assume the image
+is there.
+
+Solution: start the funko on the server. Done. It's implemented.
+
+## Variant 3: Deploy to the server and it's already running
+
+1. If it's already running and it's running the latest image, then nothing
+   to be done.
+2. It it's running and is not the latest, we can stop it and start with the
+   latest image.
+
+* Action 2 causes downtime. Usually it will not be significant, but it's there.
+* In the future it may be important to have zero downtime.
+* We need to figure out what is implied by doing "zero downtime" to see if
+  not doing it now would make it impossible.
+
+For zero downtime, we want to have two instances running, switch the proxy
+to the new one, then stop the old one.
+
+Currently it's impossible to run two instances because the container name is
+faaso-funkoname, and we can't have 2 of those.
+
+So: we could instead have faaso-funkoname-1, faaso-funkoname-2, etc.
+with some sort of suffix
+
+Changes implied in the faaso code:
+
+* If we have two containers for one funko, we need to consider the "state" of
+  the funko differently.
+* What does it mean to start/pause/stop a funko with two instances
+* Do we want to enable two-instance funkos? With round-robin proxy?
+* What happens if we have two instances with different images?
+
+Answers coming up.

docs/funkos.md

@@ -49,10 +49,12 @@ faaso-net.
 The proxy is the only container exposed to the host network, everything
 else needs to be accessed through it.
 
-The faaso-proxy container will automatically proxy all requests so if you access the URL `http://faaso-proxy:8888/funko/hello/foo` that will be
+The faaso-proxy container will automatically proxy all requests so if
+you access the URL `http://faaso-proxy:8888/funko/hello/foo` that will be
 proxied to `/foo` in the `hello` funko.
 
-This is all done via naming conventions. You can create your own `faaso-whatever` container, add it to the `faaso-net` and faaso will
+This is all done via naming conventions. You can create your own
+`faaso-whatever` container, add it to the `faaso-net` and faaso will
 happily consider it a funko.
 
 In the same way all funkos are simply docker containers running in that
@@ -78,4 +80,3 @@ faaso-proxy -- GET /bar --> faaso-funko1
 The dynamic proxying is achieved by reading the current state of
 Docker and just adapt to it using the naming conventions mentioned
 above.
-

docs/secrets.md

@@ -31,7 +31,7 @@ So, the proxy can periodically examine its secret store and populate a folder
 If on starting a funko we always do a bind mount of `/secrets/foo` to `/secrets`
 then it will always have its secrets in place.
 
-## Problem 2: how can the proxy know the secrets without keeping them in the image?
+## Problem 2: how can the proxy know the secrets without keeping them in the image
 
 They can't be shipped via the image, so they need to be injected via the admin API.
 

examples/hello_crystal/README.md

@@ -0,0 +1,11 @@
+# Readme for Hello_crystal
+
+This is a funko using the Crystal runtime for [FaaSO](https://git.ralsina.me/ralsina/faaso)
+
+## What is Hello_crystal
+
+Write here what it is
+
+## How to use Hello_crystal
+
+And so on.

examples/hello_crystal/funko.cr

@@ -1,3 +1,16 @@
+require "kemal"
+
+# This is a kemal app, you can add handlers, middleware, etc.
+
+# A basic hello world get endpoint
 get "/" do
   "Hello World Crystal!"
 end
+
+# The `/ping/` endpoint is configured in the container as a healthcheck
+# You can make it better by checking that your database is responding
+# or whatever checks you think are important
+#
+get "/ping/" do
+  "OK"
+end

examples/hello_crystal/shard.yml

@@ -1,4 +1,4 @@
-name: function
+name: hello_crystal
 version: 0.1.0
 
 targets:
@@ -12,4 +12,3 @@ dependencies:
 # development_dependencies:
 #   webmock:
 #     github: manastech/webmock.cr
-

public/bars.svg

@@ -0,0 +1,52 @@
+<svg width="16" height="16" viewBox="0 0 135 140" xmlns="http://www.w3.org/2000/svg" fill="#494949">
+    <rect y="10" width="15" height="120" rx="6">
+        <animate attributeName="height"
+             begin="0.5s" dur="1s"
+             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
+             repeatCount="indefinite" />
+        <animate attributeName="y"
+             begin="0.5s" dur="1s"
+             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
+             repeatCount="indefinite" />
+    </rect>
+    <rect x="30" y="10" width="15" height="120" rx="6">
+        <animate attributeName="height"
+             begin="0.25s" dur="1s"
+             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
+             repeatCount="indefinite" />
+        <animate attributeName="y"
+             begin="0.25s" dur="1s"
+             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
+             repeatCount="indefinite" />
+    </rect>
+    <rect x="60" width="15" height="140" rx="6">
+        <animate attributeName="height"
+             begin="0s" dur="1s"
+             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
+             repeatCount="indefinite" />
+        <animate attributeName="y"
+             begin="0s" dur="1s"
+             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
+             repeatCount="indefinite" />
+    </rect>
+    <rect x="90" y="10" width="15" height="120" rx="6">
+        <animate attributeName="height"
+             begin="0.25s" dur="1s"
+             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
+             repeatCount="indefinite" />
+        <animate attributeName="y"
+             begin="0.25s" dur="1s"
+             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
+             repeatCount="indefinite" />
+    </rect>
+    <rect x="120" y="10" width="15" height="120" rx="6">
+        <animate attributeName="height"
+             begin="0.5s" dur="1s"
+             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
+             repeatCount="indefinite" />
+        <animate attributeName="y"
+             begin="0.5s" dur="1s"
+             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
+             repeatCount="indefinite" />
+    </rect>
+</svg>

public/index.html

@@ -1,25 +1,106 @@
 <!DOCTYPE html>
+
 <head>
   <meta charset="utf-8" />
-  <link rel="stylesheet" href="https://matcha.mizu.sh/matcha.css" />
-  <script src="https://unpkg.com/htmx.org@2.0.0"></script>
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@1/css/pico.min.css" />
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@2/css/pico.colors.min.css" />  <script src="https://unpkg.com/htmx.org@2.0.0"></script>
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <body>
+</head>
+
+<body>
   <header class="container">
-    <h1>Your Funko Collection</h1>
-    </header>
-    <main class=container>
-    <button hx-get="funkos/?format=html" hx-target="#funko-list">Update</button>
-    <table>
-    <thead>
-      <tr>
-        <th>Name</th>
-        <th>Status</th>
-        <th>Action</th>
-      </tr>
+    <h1>FaaSO Admin Interface</h1>
+  </header>
+  <article>
+    <nav>
+      <ul>
+      <li><strong style="font-size: 200%;">Your Funko Collection</strong></li>
+      </ul>
+      <ul>
+      <li><button id="update-funkos" style="float:right; display:inline;" hx-trigger="load, click, every 60s"
+        hx-get="funkos/?format=html" hx-target="#funko-list">
+        Refresh
+      </button>
+      </ul>
+    </nav>
+    <span id="message"></span>
+    <table hx-target="#message" class="striped">
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Instances</th>
+          <th>Actions</th>
+        </tr>
+      </thead>
       <tbody id="funko-list">
       </tbody>
-    </thead>
-    </main>
-  </body>
-</head>
+    </table>
+    <div id="terminal" style="resize: vertical; overflow: auto;"></div>
+  </article>
+  <script>
+    update_funkos = function () {
+      document.getElementById("update-funkos").click()
+    }
+  </script>
+  <article>
+    <nav>
+    <ul>
+    <li><strong style="font-size: 200%;">Your Secrets</strong>
+    </ul>
+    <ul>
+      <li><button id="update-secrets" style="float:right; display:inline;" hx-trigger="load, click, every 60s"
+        hx-get="secrets/?format=html" hx-target="#secret-list">
+        Refresh
+      </button>
+      <li><button style="float:right; display:inline;" onclick="show_new_secret()">
+        Add
+      </button>
+    </ul>
+    </nav>
+    <span id="message"></span>
+    <table hx-target="#message">
+      <thead>
+        <tr>
+          <th>Funko</th>
+          <th>Name</th>
+          <th>Actions</th>
+        </tr>
+      </thead>
+      <tbody id="secret-list">
+      </tbody>
+      <dialog id="add-secret">
+      <article>
+      <header>
+        New Secret
+      </header>
+        <form hx-post="secrets/">
+          <input placeholder="funko name" id="new-secret-funko" name="funko">
+          <input placeholder="secret name" id="new-secret-name" name="name">
+          <input placeholder="secret value" type="password" id="new-secret-password" name="value">
+          <fieldset role="group" style="text-align: right;">
+            <button style="width:9em; display: inline;" type="submit" hx-on:htmx:after-request="hide_new_secret()">CREATE</button>
+            <button style="width:9em; display: inline;" onclick="hide_new_secret(); close();" aria-label="Close" rel="prev">CLOSE</button>
+          </fieldset>
+        </form>
+      </article>
+      </dialog>
+      <script>
+        update_secrets = function() {
+          document.getElementById("update-secrets").click()
+        }
+        show_new_secret = function() {
+          document.getElementById('new-secret-funko').value=""
+          document.getElementById('new-secret-name').value=""
+          document.getElementById('new-secret-password').value=""
+          document.getElementById('add-secret').show()
+        }
+        hide_new_secret = function() {
+          document.getElementById('new-secret-funko').value=""
+          document.getElementById('new-secret-name').value=""
+          document.getElementById('new-secret-password').value=""
+          document.getElementById('add-secret').close()
+          update_secrets()
+        }
+      </script>
+  </article>
+</body>

runtimes/crystal/Dockerfile.j2

@@ -10,7 +10,7 @@ RUN shards build --release
 RUN strip bin/*
 
 FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine as ship
-RUN apk update && apk upgrade && apk add openssl pcre2 libgcc gc libevent curl {{ ship_packages | join " " }} && apk cache clean
+RUN apk update && apk upgrade && apk add pcre2 libgcc gc libevent curl {{ ship_packages | join " " }} && apk cache clean
 RUN addgroup -S app && adduser app -S -G app
 
 WORKDIR /home/app
@@ -19,4 +19,4 @@ USER app
 COPY --from=build /home/app/bin/funko  .
 
 CMD ["./funko"]
-HEALTHCHECK {{ healthcheck_options }}  CMD {{ healthcheck_command }}
+HEALTHCHECK {{ healthcheck_options }}  CMD {{ healthcheck_command }}

runtimes/crystal/README.md

@@ -0,0 +1,4 @@
+# README
+
+This is the readme for people wanting to change this runtime,
+not for people trying to use it

runtimes/crystal/main.cr

@@ -1,8 +1,4 @@
 require "kemal"
 require "./funko.cr"
 
-get "/ping/" do
-  "OK"
-end
-
 Kemal.run

runtimes/crystal/template/README.md.j2

@@ -0,0 +1,11 @@
+# Readme for {{name | title}}
+
+This is a funko using the Crystal runtime for [FaaSO](https://git.ralsina.me/ralsina/faaso)
+
+## What is {{name | title}}
+
+Write here what it is
+
+## How to use {{name | title}}
+
+And so on.

runtimes/crystal/template/funko.cr

@@ -0,0 +1,16 @@
+require "kemal"
+
+# This is a kemal app, you can add handlers, middleware, etc.
+
+# A basic hello world get endpoint
+get "/" do
+  "Hello World Crystal!"
+end
+
+# The `/ping/` endpoint is configured in the container as a healthcheck
+# You can make it better by checking that your database is responding
+# or whatever checks you think are important
+#
+get "/ping/" do
+  "OK"
+end

runtimes/crystal/template/funko.yml.j2

@@ -0,0 +1,2 @@
+name: {{ name }}
+runtime: {{ runtime }}

runtimes/crystal/template/shard.yml.j2

@@ -0,0 +1,14 @@
+name: {{ name }}
+version: 0.1.0
+
+targets:
+  funko:
+    main: main.cr
+
+dependencies:
+  kemal:
+    github: kemalcr/kemal
+
+# development_dependencies:
+#   webmock:
+#     github: manastech/webmock.cr

runtimes/express/Dockerfile.j2

@@ -0,0 +1,20 @@
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine as build
+
+RUN apk update && apk upgrade && apk add nodejs npm {{ ship_packages | join(" ") }} {{ devel_packages | join(" ") }} && apk cache clean
+
+WORKDIR /home/app
+
+COPY ./ ./
+RUN npm i
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine as ship
+RUN apk update && apk upgrade && apk add nodejs curl {{ ship_packages | join " " }} && apk cache clean
+RUN addgroup -S app && adduser app -S -G app
+
+WORKDIR /home/app
+USER app
+
+COPY --from=build /home/app/  .
+
+CMD ["node", "funko.js"]
+HEALTHCHECK {{ healthcheck_options }}  CMD {{ healthcheck_command }}

runtimes/express/template/funko.js

@@ -0,0 +1,15 @@
+const express = require('express')
+const app = express()
+const port = 3000
+
+app.get('/', (req, res) => {
+    res.send('Hello World!')
+})
+
+app.get('/ping', (req, res) => {
+    res.send('OK')
+})
+
+app.listen(port, () => {
+    console.log(`Example funko listening on port ${port}`)
+})

runtimes/express/template/funko.yml.j2

@@ -0,0 +1,2 @@
+name: {{ name }}
+runtime: {{ runtime }}

runtimes/express/template/package.json.j2

@@ -0,0 +1,11 @@
+{
+  "name": "{{name}}",
+  "version": "1.0.0",
+  "main": "funko.js",
+  "author": "",
+  "license": "MIT",
+  "description": "Example Funko",
+  "dependencies": {
+    "express": "^4.19.2"
+  }
+}

runtimes/flask/Dockerfile.j2

@@ -0,0 +1,22 @@
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine as build
+
+RUN apk update && apk upgrade && apk add python3 gcc musl-dev linux-headers python3-dev {{ ship_packages | join(" ") }} {{ devel_packages | join(" ") }} && apk cache clean
+
+WORKDIR /home/app
+
+COPY requirements.txt *.py ./
+RUN python3 -m venv venv
+RUN venv/bin/pip install uwsgi
+RUN venv/bin/pip install -r requirements.txt
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine as ship
+RUN apk update && apk upgrade && apk add python3 uwsgi curl {{ ship_packages | join " " }} && apk cache clean
+RUN addgroup -S app && adduser app -S -G app
+
+WORKDIR /home/app
+USER app
+
+COPY --from=build /home/app/  .
+
+CMD ["venv/bin/uwsgi", "--http", "0.0.0.0:3000", "--master", "-p", "1", "-w", "funko:app"]
+HEALTHCHECK {{ healthcheck_options }}  CMD {{ healthcheck_command }}

runtimes/flask/main.py

@@ -0,0 +1,6 @@
+from flask import Flask
+
+app = Flask({{name}})
+
+if __name__ == '__main__':
+    serve(app, host='0.0.0.0', port=5000)

runtimes/flask/template/funko.py.j2

@@ -0,0 +1,11 @@
+from flask import Flask
+
+app = Flask("{{name}}")
+
+@app.route('/')
+def handle(req):
+    return "Hello World from Flask!"
+
+@app.route('/ping')
+def handle(req):
+    return "OK"

runtimes/flask/template/funko.yml.j2

@@ -0,0 +1,2 @@
+name: {{ name }}
+runtime: {{ runtime }}

runtimes/flask/template/requirements.txt

@@ -0,0 +1 @@
+flask

shard.lock

@@ -4,9 +4,13 @@ shards:
     git: https://github.com/sija/backtracer.cr.git
     version: 1.2.2
 
-  commander:
-    git: https://github.com/mrrooijen/commander.git
-    version: 0.4.0
+  base58:
+    git: https://github.com/crystal-china/base58.cr.git
+    version: 0.1.0+git.commit.d1150d4a6f086013a475640ad00e561a2fe1082a
+
+  cr-config:
+    git: https://github.com/crystal-community/cr-config.git
+    version: 5.1.0+git.commit.5eae3dfbf97da7dfa7c6e64a2a508069948518d3
 
   crest:
     git: https://github.com/mamantoha/crest.git
@@ -20,9 +24,13 @@ shards:
     git: https://github.com/naqvis/crystar.git
     version: 0.4.0
 
+  docopt:
+    git: https://github.com/chenkovsky/docopt.cr.git
+    version: 0.2.0+git.commit.620fce4f334ff15d7321e5ecb6665ad258fe9297
+
   docr:
     git: https://github.com/ralsina/docr.git
-    version: 0.1.1+git.commit.18f15cc7111b1d0c63347c7cca07aee9ec87a7a8
+    version: 0.1.1+git.commit.98a20178d5ae1391f1cd56e372530de6aa2b1ebc
 
   exception_page:
     git: https://github.com/crystal-loot/exception_page.git
@@ -44,7 +52,14 @@ shards:
     git: https://github.com/kemalcr/kemal-basic-auth.git
     version: 1.0.0
 
+  oplog:
+    git: https://github.com/ralsina/oplog.git
+    version: 0.1.0+git.commit.70e3a7bbc2f1f4d75cf4e142244b263ee2844ba1
+
   radix:
     git: https://github.com/luislavena/radix.git
     version: 0.4.1
 
+  rucksack:
+    git: https://github.com/busyloop/rucksack.git
+    version: 2.0.0

shard.yml

@@ -15,18 +15,29 @@ crystal: ">= 1.12.2"
 license: MIT
 
 dependencies:
+  base58:
+    github: crystal-china/base58.cr
+  crest:
+    github: mamantoha/crest
+  crinja:
+    github: straight-shoota/crinja
+  cr-config:
+    github: crystal-community/cr-config
+  crystar:
+    github: naqvis/crystar
+  docopt:
+    github: chenkovsky/docopt.cr
   docr:
     github: ralsina/docr
     branch: add_exposed_ports
-  commander:
-    github: mrrooijen/commander
   kemal:
     github: kemalcr/kemal
   kemal-basic-auth:
     github: kemalcr/kemal-basic-auth
-  crinja:
-    github: straight-shoota/crinja
-  crystar:
-    github: naqvis/crystar
-  crest:
-    github: mamantoha/crest    
+  oplog:
+    github: ralsina/oplog
+  rucksack:
+    github: busyloop/rucksack
+
+scripts:
+  postinstall: cat .rucksack >> bin/faaso

src/commands/build.cr

@@ -0,0 +1,72 @@
+require "base58"
+
+module Faaso
+  module Commands
+    # Build images for one or more funkos from source
+    struct Build
+      def run(options, folders : Array(String)) : Int32
+        funkos = Funko::Funko.from_paths(folders)
+        # Create temporary build location
+
+        funkos.each do |funko|
+          tmp_dir = Path.new("tmp", Random.base58(8))
+          Dir.mkdir_p(tmp_dir) unless File.exists? tmp_dir
+
+          funko.runtime = nil if options["--no-runtime"]
+
+          funko.prepare_build(path: tmp_dir)
+          if options["--local"]
+            Log.info { "Building function... #{funko.name} in #{tmp_dir}" }
+            funko.build tmp_dir
+            FileUtils.rm_rf(tmp_dir)
+            next
+          end
+          Faaso.check_version
+          # Create a tarball for the funko
+          buf = IO::Memory.new
+          Compress::Gzip::Writer.open(buf) do |gzip|
+            Crystar::Writer.open(gzip) do |tw|
+              Log.debug { "Adding files to tarball" }
+              Dir.glob("#{tmp_dir}/**/*").each do |path|
+                next unless File.file? path
+                rel_path = Path[path].relative_to tmp_dir
+                Log.debug { "Adding #{rel_path}" }
+                file_info = File.info(path)
+                hdr = Crystar::Header.new(
+                  name: rel_path.to_s,
+                  mode: file_info.permissions.to_u32,
+                  size: file_info.size,
+                )
+                tw.write_header(hdr)
+                tw.write(File.read(path).to_slice)
+              end
+            end
+          end
+          FileUtils.rm_rf(tmp_dir)
+          tmp = File.tempname
+          File.open(tmp, "w") do |outf|
+            outf << buf
+          end
+
+          url = "#{Config.server}funkos/build/"
+
+          user, password = Config.auth
+          Log.info { "Uploading funko to #{Config.server}" }
+          Log.info { "Starting remote build:" }
+          Crest.post(
+            url,
+            {"funko.tgz" => File.open(tmp), "name" => "funko.tgz"},
+            user: user, password: password
+          ) do |response|
+            IO.copy(response.body_io, STDOUT)
+          end
+          Log.info { "Build finished successfully." }
+        rescue ex : Crest::InternalServerError
+          Log.error(exception: ex) { "Error building funko #{funko.name} from #{funko.path}" }
+          return 1
+        end
+        0
+      end
+    end
+  end
+end

src/commands/deploy.cr

@@ -0,0 +1,59 @@
+module Faaso
+  module Commands
+    struct Deploy
+      def local(options, funko_name : String) : Int32
+        funko = Funko::Funko.from_names([funko_name])[0]
+        # Get scale, check for out-of-date containers
+        current_scale = funko.scale
+        latest_image = funko.latest_image
+        containers = funko.containers
+        out_of_date = containers.count { |container| container.image_id != latest_image }
+        Log.info { "Need to update #{out_of_date} containers" }
+        Log.info { "Scaling from #{current_scale} to #{current_scale + out_of_date}" }
+        # Increase scale to get enough up-to-date containers
+        new_containers = funko.scale(current_scale + out_of_date)
+
+        # Wait for them to be healthy
+        begin
+          funko.wait_for(current_scale + out_of_date, 120, healthy: true)
+        rescue ex : Exception
+          # Failed to start, rollback
+          Log.error(exception: ex) { "Failed to scale, rolling back" }
+          docker_api = Docr::API.new(Docr::Client.new)
+          new_containers.each do |container_id|
+            docker_api.containers.stop(container_id)
+            docker_api.containers.delete(container_id)
+          end
+          return 1
+        end
+
+        Log.info { "Scaling down to #{current_scale}" }
+        # Decrease scale to the desired amount
+        funko.scale(current_scale)
+        funko.wait_for(current_scale, 30)
+        Log.info { "Deployed #{funko_name}" }
+        0
+      end
+
+      def remote(options, funko_name : String) : Int32
+        user, password = Config.auth
+        Faaso.check_version
+        Crest.get(
+          "#{Config.server}funkos/#{funko_name}/deploy/", \
+             user: user, password: password) do |response|
+          IO.copy(response.body_io, STDOUT)
+        end
+        0
+      end
+
+      def run(options, funko_name : String) : Int32
+        Log.info { "Deploying #{funko_name}" }
+        if options["--local"]
+          local(options, funko_name)
+        else
+          remote(options, funko_name)
+        end
+      end
+    end
+  end
+end

src/commands/export.cr

@@ -0,0 +1,19 @@
+module Faaso
+  module Commands
+    struct Export
+      def run(options, source : String, destination : String) : Int32
+        funko = Funko::Funko.from_paths([source])[0]
+        # Create temporary build location
+        dst_path = destination
+        if File.exists? dst_path
+          Log.error { "#{dst_path} already exists, not exporting #{funko.path}" }
+          return 1
+        end
+        Log.info { "Exporting #{funko.path} to #{dst_path}" }
+        Dir.mkdir_p(dst_path)
+        funko.prepare_build Path[dst_path]
+        0
+      end
+    end
+  end
+end

src/commands/login.cr

@@ -0,0 +1,47 @@
+module Faaso
+  module Commands
+    struct Login
+      def run(options) : Int32
+        server = Config.server
+        Log.info { "Enter password for #{server}" }
+        if STDIN.tty?
+          password = (STDIN.noecho &.gets.try &.chomp).to_s
+        else
+          password = STDIN.gets.to_s
+        end
+        # This is tricky. If the service is running behind a reverse proxy
+        # then /version is locked, but if it's not, only /auth is locked.
+        # So we try /version first without a password, and if it succeeds
+        # we try /auth with the password. If /version fails, we try /version
+        # with the password
+        #
+        begin
+          # Version without password.
+          Crest.get("#{server}version/")
+          # Auth with password
+          begin
+            Crest.get("#{server}auth/", user: "admin", password: password)
+          rescue ex : Crest::Unauthorized
+            # Failed with auth/
+            Log.error { "Wrong password" }
+            return 1
+          end
+        rescue ex : Crest::Unauthorized
+          # Version with password
+          Crest.get("#{server}version/", user: "admin", password: password)
+        end
+
+        # If we got here the password is ok
+        CONFIG.hosts[server] = {"admin", password}
+        Config.save
+        0
+      rescue ex : Crest::Unauthorized
+        Log.error { "Wrong password" }
+        1
+      rescue ex : Socket::ConnectError
+        Log.error { "Connection refused" }
+        1
+      end
+    end
+  end
+end

src/commands/new.cr

@@ -0,0 +1,35 @@
+require "../runtime.cr"
+
+module Faaso
+  module Commands
+    # Creates a new empty funko out of a given runtime
+    struct New
+      def run(options, folder) : Int32
+        runtime = options["-r"].as(String)
+        # Give a list of known runtimes
+        if runtime == "list"
+          Runtime.list
+          return 0
+        end
+
+        # Create new folder
+        if Dir.exists? folder
+          Log.error { "Folder #{folder} already exists" }
+          return 1
+        end
+
+        # Get runtime template files list
+        template_base, template_files = Runtime.template_files(runtime)
+
+        Runtime.copy_templated(
+          template_base,
+          template_files,
+          folder,
+          {"name"    => Path[folder].basename,
+           "runtime" => runtime}
+        )
+        0
+      end
+    end
+  end
+end

src/commands/scale.cr

@@ -0,0 +1,60 @@
+module Faaso
+  module Commands
+    # Controls a funko's scale
+    #
+    # Scale is how many instances are running.
+    #
+    # If it's increased, more instances are created.
+    # It it's decreased, instances are destroyed.
+    #
+    # In both cases stopped instances after the required
+    # scale is reached are deleted.
+    struct Scale
+      def local(options, name : String, scale : Int | Nil) : Int32
+        funko = Funko::Funko.from_names([name])[0]
+        # Asked about scale
+        if funko.image_history.empty?
+          Log.error { "Unknown funko #{funko.name}" }
+          return 1
+        end
+        if scale.nil?
+          Log.info { "Funko #{name} has a scale of #{funko.scale}" }
+          return 0
+        end
+        # Asked to set scale
+        funko.scale(scale)
+        0
+      end
+
+      def remote(options, name : String, scale : Int | Nil) : Int32
+        user, password = Config.auth
+        Faaso.check_version
+        if scale.nil?
+          Crest.get(
+            "#{Config.server}funkos/#{name}/scale/", \
+               user: user, password: password) do |response|
+            IO.copy(response.body_io, STDOUT)
+          end
+          return 0
+        end
+        Crest.post(
+          "#{Config.server}funkos/#{name}/scale/",
+          {"scale" => scale}, user: user, password: password) do |response|
+          IO.copy(response.body_io, STDOUT)
+        end
+        0
+      rescue ex : Crest::InternalServerError
+        Log.error(exception: ex) { "Error scaling funko #{name}" }
+        1
+      end
+
+      def run(options, name : String, scale) : Int32
+        scale = scale.try &.to_s.to_i
+        if options["--local"]
+          return local(options, name, scale)
+        end
+        remote(options, name, scale)
+      end
+    end
+  end
+end

src/commands/secret.cr

@@ -0,0 +1,53 @@
+module Faaso
+  module Commands
+    struct Secret
+      def local(options, funko, name, secret) : Int32
+        if options["--add"]
+          dst_dir = "secrets/#{funko}"
+          Dir.mkdir_p(dst_dir) unless Dir.exists?(dst_dir)
+          File.write("#{dst_dir}/#{name}", secret)
+        elsif options["--delete"]
+          File.delete("secrets/#{funko}/#{name}")
+        end
+        0
+      end
+
+      def remote(options, funko, name, secret) : Int32
+        Faaso.check_version
+        user, password = Config.auth
+        if options["--add"]
+          Crest.post(
+            "#{Config.server}secrets/",
+            {
+              "funko" => funko,
+              "name"  => name,
+              "value" => secret,
+            }, user: user, password: password)
+          Log.info { "Secret created" }
+        elsif options["--delete"]
+          Crest.delete(
+            "#{Config.server}secrets/#{funko}/#{name}",
+            user: user, password: password)
+        end
+        0
+      rescue ex : Crest::RequestFailed
+        Log.error { "Error #{ex.response.status_code}" }
+        1
+      end
+
+      def run(options, funko, name) : Int32
+        if options["--add"]
+          Log.info { "Enter the secret, end with Ctrl-D" } if STDIN.tty?
+          secret = STDIN.gets_to_end
+        else
+          secret = ""
+        end
+
+        if options["--local"]
+          return local(options, funko, name, secret)
+        end
+        remote(options, funko, name, secret)
+      end
+    end
+  end
+end

src/commands/status.cr

@@ -0,0 +1,50 @@
+module Faaso
+  module Commands
+    struct Status
+      def local(options, name) : Int32
+        funko = Funko::Funko.from_names([name])[0]
+        status = funko.docker_status
+
+        if status.images.size == 0
+          Log.error { "Unkown funko: #{name}" }
+          return 1
+        end
+
+        Log.info { "Name: #{status.@name}" }
+        Log.info { "Scale: #{status.scale}" }
+
+        Log.info { "Containers: #{status.containers.size}" }
+        status.containers.each do |container|
+          Log.info { "  #{container.@names[0]} #{container.status}" }
+        end
+
+        Log.info { "Images: #{status.images.size}" }
+        status.images.each do |image|
+          Log.info { "  #{image.repo_tags} #{Time.unix(image.created)}" }
+        end
+        0
+      end
+
+      def remote(options, name) : Int32
+        Faaso.check_version
+        user, password = Config.auth
+        Crest.get(
+          "#{Config.server}funkos/#{name}/status/", \
+             user: user, password: password) do |response|
+          IO.copy(response.body_io, STDOUT)
+        end
+        0
+      rescue ex : Crest::InternalServerError
+        Log.error(exception: ex) { "Error scaling funko #{name}" }
+        1
+      end
+
+      def run(options, name) : Int32
+        if options["--local"]
+          return local(options, name)
+        end
+        remote(options, name)
+      end
+    end
+  end
+end

src/config.cr

@@ -0,0 +1,44 @@
+require "yaml"
+
+CONFIG = Config.load
+
+class Config
+  include YAML::Serializable
+
+  property hosts : Hash(String, {String, String}) = Hash(String, {String, String}).new
+
+  def initialize
+    @hosts = {} of String => {String, String}
+  end
+
+  def self.load : Config
+    if File.file? ".faaso.yml"
+      return Config.from_yaml(File.read(".faaso.yml"))
+    end
+    Config.new
+  end
+
+  def self.save
+    File.open(".faaso.yml", "w") do |outf|
+      outf << CONFIG.to_yaml
+    end
+  end
+
+  @@already_warned = false
+
+  def self.server : String
+    @@already_warned = true
+    url = ENV.fetch("FAASO_SERVER", nil)
+    if url.nil?
+      Log.warn { "FAASO_SERVER not set" } unless @@already_warned
+      url = "http://localhost:3000/"
+    end
+    url += "/" unless url.ends_with? "/"
+    Log.info { "Using server #{url}" } unless @@already_warned
+    url
+  end
+
+  def self.auth : {String, String}
+    CONFIG.hosts.fetch(server, {"admin", ""})
+  end
+end

src/daemon/config.cr

@@ -0,0 +1,47 @@
+require "cr-config"
+require "kemal-basic-auth"
+
+class Config
+  include CrConfig
+
+  option password : String, default: "admin"
+
+  def self.load
+    builder = Config.new_builder
+    builder.providers do
+      [
+        CrConfig::Providers::SimpleFileProvider.new("config/faaso.yml"),
+        CrConfig::Providers::EnvVarProvider.new,
+      ]
+    end
+    config = builder.build
+    Config.set_instance config
+  end
+end
+
+class ConfigAuthHandler < Kemal::BasicAuth::Handler
+  only ["/auth", "/auth/*"]
+
+  def call(context)
+    return call_next(context) unless only_match?(context)
+    super
+  end
+
+  def initialize
+    # Ignored, just make the compiler happy
+    @credentials = Kemal::BasicAuth::Credentials.new({"foo" => "bar"})
+  end
+
+  def authorize?(value) : String?
+    username, password = Base64.decode_string(value[BASIC.size + 1..-1]).split(":")
+    if username == "admin" && password == Config.instance.password
+      username
+    else
+      nil
+    end
+  end
+end
+
+# Tie auth to config
+
+add_handler ConfigAuthHandler.new

src/daemon/funko.cr

@@ -1,3 +1,4 @@
+require "base58"
 require "docr"
 require "kemal"
 require "../funko.cr"
@@ -5,26 +6,71 @@ require "../funko.cr"
 module Funko
   extend self
 
+  # Get the funko's status
+  get "/funkos/:name/status/" do |env|
+    name = env.params.url["name"]
+    run_faaso(["status", name], env)
+  end
+
+  # Get the funko's scale
+  get "/funkos/:name/scale/" do |env|
+    name = env.params.url["name"]
+    run_faaso(["scale", name], env)
+  end
+
+  # Set the funko's scale
+  post "/funkos/:name/scale/" do |env|
+    name = env.params.url["name"]
+    scale = env.params.body["scale"].as(String)
+    run_faaso(["scale", name, scale], env)
+  end
+
+  get "/funkos/:name/deploy" do |env|
+    name = env.params.url["name"]
+    run_faaso(["deploy", name], env)
+  end
+
+  # Build image for funko received as "funko.tgz"
+  # TODO: This may take a while, consider using something like
+  # mosquito-cr/mosquito to make it a job queue
+  post "/funkos/build/" do |env|
+    # Create place to build funko
+    tmp_dir = Path.new("tmp", Random.base58(8))
+    Dir.mkdir_p(tmp_dir) unless File.exists? tmp_dir
+
+    # Expand tarball in there
+    file = env.params.files["funko.tgz"].tempfile
+    Compress::Gzip::Reader.open(file) do |gzip|
+      Crystar::Reader.open(gzip) do |tar|
+        tar.each_entry do |entry|
+          dst = Path.new(tmp_dir, entry.name)
+          Dir.mkdir_p dst.dirname
+          File.open(Path.new(tmp_dir, entry.name), "w") do |outf|
+            IO.copy entry.io, outf
+          end
+        end
+      end
+    end
+
+    # Build the thing
+    run_faaso(["build", tmp_dir.to_s, "--no-runtime"], env)
+  ensure
+    FileUtils.rm_rf(tmp_dir) unless tmp_dir.nil?
+  end
+  # Endpoints for the web frontend
+
+  # General status for the front page
   get "/funkos/" do |env|
     funkos = Funko.from_docker
     funkos.sort! { |a, b| a.name <=> b.name }
-    result = [] of Hash(String, String)
+    result = [] of Hash(String, String | Array(Docr::Types::ContainerSummary))
 
     funkos.each do |funko|
-      state = ""
-      case funko
-      when .running?
-        state = "running"
-      when .paused?
-        state = "paused"
-      else
-        state = "stopped"
-      end
-
       result << {
-        "name"   => funko.name,
-        "state"  => state,
-        "status" => funko.status,
+        "name"         => funko.name,
+        "scale"        => funko.scale.to_s,
+        "containers"   => funko.containers,
+        "latest_image" => funko.latest_image,
       }
     end
 
@@ -34,4 +80,76 @@ module Funko
       result.to_json
     end
   end
+  # Stop => scale to 0
+  get "/funkos/:name/stop" do |env|
+    name = env.params.url["name"]
+    funko = Funko.from_names([name])[0]
+    funko.scale(0)
+    funko.wait_for(0, 1)
+  end
+
+  # Start => scale to 1
+  get "/funkos/:name/start" do |env|
+    name = env.params.url["name"]
+    funko = Funko.from_names([name])[0]
+    if funko.scale == 0
+      funko.scale(1)
+      funko.wait_for(1, 1)
+    end
+  end
+
+  # Restart => scale to 0, then 1
+  get "/funkos/:name/restart" do |env|
+    name = env.params.url["name"]
+    funko = Funko.from_names([name])[0]
+    funko.scale(0)
+    funko.wait_for(0, 1)
+    funko.scale(1)
+    funko.wait_for(1, 1)
+  end
+
+  # Delete => scale to 0, remove all containers and images
+  delete "/funkos/:name/" do |env|
+    name = env.params.url["name"]
+    funko = Funko.from_names([name])[0]
+    funko.scale(0)
+    funko.wait_for(0, 1)
+    funko.remove_all_containers
+    funko.remove_all_images
+  end
+
+  # Return an iframe that shows the container's logs
+  get "/funkos/terminal/logs/:instance/" do |env|
+    instance = env.params.url["instance"]
+    Terminal.start_terminal(["docker", "logs", "-f", instance])
+    "<iframe src='terminal/' width='100%' height='100%'></iframe>"
+  end
+
+  # Get an iframe with a shell into the container
+  get "/funkos/terminal/shell/:instance/" do |env|
+    instance = env.params.url["instance"]
+    Terminal.start_terminal(["docker", "exec", "-ti", instance, "/bin/sh"], readonly: false)
+    "<iframe src='terminal/' width='100%' height='100%'></iframe>"
+  end
+
+  # Helper to run faaso locally and respond via env
+  def run_faaso(args : Array(String), env)
+    args << "-l" # Always local in the server
+    Log.info { "Running faaso [#{args}" }
+    Process.run(
+      command: "faaso",
+      args: args,
+      env: {"FAASO_SERVER_SIDE" => "true"},
+    ) do |process|
+      loop do
+        data = process.output.gets(chomp: false)
+        env.response.print data
+        env.response.flush
+        Fiber.yield # Without this the process never ends
+        break if process.terminated?
+      end
+    end
+    # FIXME: find a way to raise an exception on failure
+    # of the faaso process
+  end
 end

src/daemon/main.cr

@@ -1,72 +1,35 @@
+require "./config.cr"
 require "./funko.cr"
-require "./proxyconf.cr"
+require "./proxy.cr"
 require "./secrets.cr"
+require "./terminal.cr"
 require "compress/gzip"
 require "crystar"
 require "docr"
-require "kemal-basic-auth"
 require "kemal"
 require "uuid"
 
-# FIXME: make configurable
-basic_auth "admin", "admin"
+Config.load
 
-# Bring up the funko
-get "/funko/:name/up/" do |env|
-  name = env.params.url["name"]
-  response = run_faaso(["up", name])
-
-  if response["exit_code"] != 0
-    halt env, status_code: 500, response: response.to_json
-  else
-    response.to_json
-  end
+macro version
+  "{{ `grep version shard.yml | cut -d: -f2` }}".strip()
 end
 
-# Build image for funko received as "funko.tgz"
-# TODO: This may take a while, consider using something like
-# mosquito-cr/mosquito to make it a job queue
-post "/funko/build/" do |env|
-  # Create place to build funko
-  tmp_dir = Path.new("tmp", UUID.random.to_s)
-  Dir.mkdir_p(tmp_dir) unless File.exists? tmp_dir
-
-  # Expand tarball in there
-  file = env.params.files["funko.tgz"].tempfile
-  Compress::Gzip::Reader.open(file) do |gzip|
-    Crystar::Reader.open(gzip) do |tar|
-      tar.each_entry do |entry|
-        File.open(Path.new(tmp_dir, entry.name), "w") do |dst|
-          IO.copy entry.io, dst
-        end
-      end
-    end
-  end
-
-  # Build the thing
-  response = run_faaso(["build", tmp_dir.to_s])
-
-  if response["exit_code"] != 0
-    halt env, status_code: 500, response: response.to_json
-  else
-    response.to_json
-  end
+get "/" do |env|
+  env.redirect "/index.html"
 end
 
-def run_faaso(args : Array(String))
-  stderr = IO::Memory.new
-  stdout = IO::Memory.new
-  status = Process.run(
-    command: "faaso",
-    args: args + ["-l"], # Always local in the server
-    output: stdout,
-    error: stderr,
-  )
-  {
-    "exit_code" => status.exit_code,
-    "stdout"    => stdout.to_s,
-    "stderr"    => stderr.to_s,
-  }
+get "/version" do
+  "#{version}"
+end
+
+get "/auth" do
+end
+
+get "/reload" do
+  Log.info { "Reloading configuration" }
+  Config.load
+  "Config reloaded"
 end
 
 Kemal.run

src/daemon/proxy.cr

@@ -0,0 +1,64 @@
+require "./funko.cr"
+require "docr"
+require "kemal"
+
+module Proxy
+  CADDY_CONFIG_PATH = "config/Caddyfile"
+  CADDY_CONFIG_FUNKOS = "config/funkos"
+  @@current_config = File.read(CADDY_CONFIG_FUNKOS)
+
+  # Get current proxy config
+  get "/proxy/" do
+    @@current_config
+  end
+
+  # Bump proxy config to current docker state, returns
+  # new proxy config
+  patch "/proxy/" do
+    Log.info { "Updating routing" }
+    # Get all the funkos, create routes for them all
+    update_proxy_config
+  end
+
+  def self.update_proxy_config : Nil
+    docker_api = Docr::API.new(Docr::Client.new)
+    containers = docker_api.containers.list(all: true)
+
+    config = ""
+    funkos = Funko::Funko.from_docker
+    funkos.each do |funko|
+      next if funko.name == "proxy"
+      containers = funko.containers
+      next if containers.empty?
+      funko_urls = containers.map { |container|
+        "http://#{container.names[0].lstrip("/")}:3000"
+      }
+      config += %(
+      handle_path /faaso/#{funko.name}/* {
+        reverse_proxy /* #{funko_urls.join(" ")} {
+          health_uri /ping
+          fail_duration 30s
+        }
+      }
+      )
+    end
+
+    if @@current_config != config
+      Log.info { "Updating proxy config" }
+      File.open(CADDY_CONFIG_FUNKOS, "w") do |file|
+        file << config
+      end
+      # Reload config
+      @@current_config = config
+      Process.run(command: "caddy", args: ["reload", "--config", CADDY_CONFIG_PATH])
+    end
+  end
+end
+
+# Update proxy config every 1 second (if changed)
+spawn do
+  loop do
+    Proxy.update_proxy_config
+    sleep 1.second
+  end
+end

src/daemon/proxyconf.cr

@@ -1,62 +0,0 @@
-require "docr"
-require "kemal"
-
-module Proxy
-  @@current_config = File.read("tinyproxy.conf")
-
-  # Get current proxy config
-  get "/proxy/" do
-    @@current_config
-  end
-
-  # Bump proxy config to current docker state, returns
-  # new proxy config
-  patch "/proxy/" do
-    Log.info { "Updating routing" }
-    # Get all the funkos, create routes for them all
-    update_proxy_config
-  end
-
-  def self.update_proxy_config
-    docker_api = Docr::API.new(Docr::Client.new)
-    containers = docker_api.containers.list(all: true)
-
-    funkos = [] of String
-    containers.each { |container|
-      names = container.names.select &.starts_with? "/faaso-"
-      next if names.empty?
-      funkos << names[0][7..]
-    }
-    funkos.sort!
-
-    config = %(
-  User nobody
-  Group nogroup
-  Port 8888
-  Listen 0.0.0.0
-  Timeout 600
-  Allow 0.0.0.0/0
-  ReverseOnly Yes
-  ReverseMagic Yes
-  ReversePath "/admin/" "http://127.0.0.1:3000/"
-  ) + funkos.map { |funko| %(ReversePath "/faaso/#{funko}/" "http://#{funko}:3000/") }.join("\n")
-
-    if @@current_config != config
-      File.open("tinyproxy.conf", "w") do |file|
-        file << config
-      end
-      # Reload config
-      Process.run(command: "/usr/bin/killall", args: ["-USR1", "tinyproxy"])
-      @@current_config = config
-    end
-    config
-  end
-end
-
-# Update proxy config once a second
-spawn do
-  loop do
-    Proxy.update_proxy_config
-    sleep 1.second
-  end
-end

src/daemon/secrets.cr

@@ -7,21 +7,37 @@ module Secrets
 
   # Gets a secret in form {"name": "funko_name-secret_name", "value": "secret_value"}
   post "/secrets/" do |env|
-    name = env.params.json["name"].as(String)
-    value = env.params.json["value"].as(String)
-    SECRETS[name] = value
+    funko = env.params.body["funko"].as(String)
+    name = env.params.body["name"].as(String)
+    value = env.params.body["value"].as(String)
+    if funko.empty? || name.empty? || value.empty?
+      halt env, status_code: 400, response: "Bad request"
+    end
+    SECRETS["#{funko}-#{name}"] = value
     Secrets.update_secrets
     halt env, status_code: 201, response: "Created"
   end
 
   get "/secrets/" do |env|
-    halt env, status_code: 200, response: SECRETS.keys.to_json
+    result = [] of Hash(String, String)
+    SECRETS.each { |k, _|
+      result << {
+        "funko" => k.split("-")[0],
+        "name"  => k.split("-", 2)[1],
+      }
+    }
+    if env.params.query.fetch("format", "json") == "html"
+      render "src/views/secrets.ecr"
+    else
+      result.to_json
+    end
   end
 
   # Deletes a secret from the disk and memory
-  delete "/secrets/:name/" do |env|
+  delete "/secrets/:funko/:name/" do |env|
+    funko = env.params.url["funko"]
     name = env.params.url["name"]
-    SECRETS.delete(name)
+    SECRETS.delete("#{funko}-#{name}")
     update_secrets
     halt env, status_code: 204, response: "Deleted"
   end

src/daemon/terminal.cr

@@ -0,0 +1,21 @@
+module Terminal
+  extend self
+
+  @@terminal_process : Process | Nil = nil
+
+  def start_terminal(_args = ["sh"], readonly = true)
+    args = ["-p", "7681", "-o"]
+    args += ["-W"] unless readonly
+    args += _args
+    # We have a process there, kill it
+    begin
+      @@terminal_process.as(Process).terminate if !@@terminal_process.nil?
+    rescue e : RuntimeError
+      Log.error { "Error terminating terminal process: #{e.message}" }
+    end
+    @@terminal_process = Process.new(
+      command: "/usr/bin/ttyd",
+      args: args)
+    Log.info { "Terminal started on port 7681" }
+  end
+end

src/faaso.cr

@@ -1,14 +1,18 @@
+require "./commands/build.cr"
+require "./commands/deploy.cr"
+require "./commands/export.cr"
+require "./commands/login.cr"
+require "./commands/new.cr"
+require "./commands/scale.cr"
+require "./commands/secret.cr"
+require "./commands/status.cr"
 require "./funko.cr"
-require "commander"
 require "crest"
 require "docr"
 require "docr/utils.cr"
 require "json"
 require "uuid"
 
-# API if you just ran faaso-daemon
-FAASO_SERVER = ENV.fetch("FAASO_SERVER", "http://localhost:3000/")
-
 # Functions as a Service, Ops!
 module Faaso
   VERSION = "0.1.0"
@@ -23,221 +27,19 @@ module Faaso
     ))
   rescue ex : Docr::Errors::DockerAPIError
     raise ex if ex.status_code != 409 # Network already exists
-
   end
 
-  module Commands
-    # Build images for one or more funkos
-    class Build
-      @arguments : Array(String) = [] of String
-      @options : Commander::Options
+  # Compare version with server's
+  def self.check_version
+    user, password = Config.auth
+    server_version = Crest.get(
+      "#{Config.server}version/", \
+         user: user, password: password).body
 
-      def initialize(options, arguments)
-        @options = options
-        @arguments = arguments
-      end
+    local_version = "#{version}"
 
-      def run
-        funkos = Funko::Funko.from_paths(@arguments)
-        local = @options.@bool["local"]
-
-        if local
-          funkos.each do |funko|
-            # Create temporary build location
-            tmp_dir = Path.new("tmp", UUID.random.to_s)
-            Dir.mkdir_p(tmp_dir) unless File.exists? tmp_dir
-            funko.prepare_build tmp_dir
-
-            Log.info { "Building function... #{funko.name} in #{tmp_dir}" }
-            funko.build tmp_dir
-          end
-        else # Running against a server
-          funkos.each do |funko|
-            # Create a tarball for the funko
-            buf = IO::Memory.new
-            Compress::Gzip::Writer.open(buf) do |gzip|
-              Crystar::Writer.open(gzip) do |tw|
-                Dir.glob("#{funko.path}/**/*").each do |path|
-                  next unless File.file? path
-                  rel_path = Path[path].relative_to funko.path
-                  file_info = File.info(path)
-                  hdr = Crystar::Header.new(
-                    name: rel_path.to_s,
-                    mode: file_info.permissions.to_u32,
-                    size: file_info.size,
-                  )
-                  tw.write_header(hdr)
-                  tw.write(File.read(path).to_slice)
-                end
-              end
-            end
-
-            tmp = File.tempname
-            File.open(tmp, "w") do |outf|
-              outf << buf
-            end
-
-            url = "#{FAASO_SERVER}funko/build/"
-
-            begin
-              Log.info { "Uploading funko to #{FAASO_SERVER}" }
-              response = Crest.post(
-                url,
-                {"funko.tgz" => File.open(tmp), "name" => "funko.tgz"},
-                user: "admin", password: "admin"
-              )
-              Log.info { "Build finished successfully." }
-              body = JSON.parse(response.body)
-              Log.info { body["stdout"] }
-            rescue ex : Crest::InternalServerError
-              Log.error { "Error building funko #{funko.name} from #{funko.path}" }
-              body = JSON.parse(ex.response.body)
-              Log.info { body["stdout"] }
-              Log.error { body["stderr"] }
-              exit 1
-            end
-          end
-        end
-      end
-    end
-
-    # Bring up one or more funkos by name.
-    #
-    # This doesn't guarantee that they will be running the latest
-    # version, and it will try to recicle paused and exited containers.
-    #
-    # If there is no other way, it will create a brand new container with
-    # the latest known image and start it.
-    #
-    # If there are no images for the funko, it will fail to bring it up.
-    class Up
-      @arguments : Array(String) = [] of String
-      @options : Commander::Options
-
-      def initialize(options, arguments)
-        @options = options
-        @arguments = arguments
-      end
-
-      def run
-        funkos = Funko::Funko.from_names(@arguments)
-        funkos.each do |funko|
-          local = @options.@bool["local"]
-
-          if !local
-            begin
-              response = Crest.get("#{FAASO_SERVER}funko/#{funko.name}/up/",
-                user: "admin", password: "admin")
-              body = JSON.parse(response.body)
-              Log.info { body["stdout"] }
-              next
-            rescue ex : Crest::InternalServerError
-              Log.error { "Error bringing up #{funko.name}" }
-              body = JSON.parse(ex.response.body)
-              Log.info { body["stdout"] }
-              Log.error { body["stderr"] }
-              exit 1
-            end
-          end
-
-          if funko.image_history.empty?
-            Log.error { "Error: no images available for #{funko.name}:latest" }
-            exit 1
-          end
-
-          case funko
-          when .running?
-            # If it's already up, do nothing
-            # FIXME: bring back out-of-date warning
-            Log.info { "#{funko.name} is already up" }
-          when .paused?
-            # If it is paused, unpause it
-            Log.info { "Resuming existing paused container" }
-            funko.unpause
-          when .exited?
-            Log.info { "Starting function #{funko.name}" }
-            Log.info { "Restarting existing exited container" }
-            funko.start
-          else
-            # Only have an image, deploy from scratch
-            Faaso.setup_network # We need it
-            Log.info { "Creating and starting new container" }
-            funko.create_container(autostart: true)
-
-            (1..5).each { |_|
-              break if funko.running?
-              sleep 0.1.seconds
-            }
-            if !funko.running?
-              Log.warn { "Container for #{funko.name} is not running yet" }
-              next
-            end
-            Log.info { "Container for #{funko.name} is running" }
-          end
-        end
-      end
-    end
-
-    class Export
-      @arguments : Array(String) = [] of String
-      @options : Commander::Options
-
-      def initialize(options, arguments)
-        @options = options
-        @arguments = arguments
-      end
-
-      def run
-        funkos = Funko::Funko.from_paths(@arguments)
-        funkos.each do |funko|
-          # Create temporary build location
-          dst_path = Path.new("export", funko.name)
-          if File.exists? dst_path
-            Log.error { "#{dst_path} already exists, not exporting #{funko.path}" }
-            next
-          end
-          Log.info { "Exporting #{funko.path} to #{dst_path}" }
-          Dir.mkdir_p(dst_path)
-          funko.prepare_build dst_path
-        end
-      end
-    end
-
-    class Down
-      @arguments : Array(String) = [] of String
-      @options : Commander::Options
-
-      def initialize(options, arguments)
-        @options = options
-        @arguments = arguments
-      end
-
-      def run
-        @arguments.each do |arg|
-          Log.info { "Stopping funko... #{arg}" }
-          # TODO: check if funko is running
-          # TODO: stop funko container
-          # TODO: delete funko container
-          # TODO: remove route from reverse proxy
-        end
-      end
-    end
-
-    class Deploy
-      @arguments : Array(String) = [] of String
-      @options : Commander::Options
-
-      def initialize(options, arguments)
-        @options = options
-        @arguments = arguments
-      end
-
-      def run
-        @arguments.each do |arg|
-          Log.info { "Deploying funko... #{arg}" }
-          # TODO: Everything
-        end
-      end
+    if server_version != local_version
+      Log.warn { "Server is version #{server_version} and client is #{local_version}" }
     end
   end
 end

src/funko.cr

@@ -1,3 +1,4 @@
+require "./runtime.cr"
 require "crinja"
 require "file_utils"
 require "yaml"
@@ -6,6 +7,20 @@ require "yaml"
 module Funko
   extend self
 
+  struct Status
+    property name : String = ""
+    property scale : Int32 = 0
+    property containers : Array(Docr::Types::ContainerSummary) = [] of Docr::Types::ContainerSummary
+    property images : Array(Docr::Types::ImageSummary) = [] of Docr::Types::ImageSummary
+
+    def initialize(name, scale, containers, images)
+      @name = name
+      @scale = scale
+      @containers = containers
+      @images = images
+    end
+  end
+
   class Funko
     include YAML::Serializable
 
@@ -25,6 +40,10 @@ module Funko
     @[YAML::Field(ignore: true)]
     property path : String = ""
 
+    # Scale: how many instances of this funko should be running
+    @[YAML::Field(ignore: true)]
+    property scale = 0
+
     # Healthcheck properties
     property healthcheck_options : String = "--interval=1m --timeout=2s --start-period=2s --retries=3"
     property healthcheck_command : String = "curl --fail http://localhost:3000/ping || exit 1"
@@ -60,30 +79,52 @@ module Funko
         }
     end
 
-    # Create an array of funkos just from names. These are limited in function
-    # and can't call `prepare_build` or some other functionality
-    def self.from_names(names : Array(String)) : Array(Funko)
-      names.map { |name|
-        Funko.from_yaml("name: #{name}")
-      }
+    # Get the number of running instances of this funko
+    def scale
+      containers.size
     end
 
-    # Get all the funkos docker knows about.
-    def self.from_docker : Array(Funko)
+    # Set the number of running instances of this funko
+    # Returns the list of IDs started or stopped
+    def scale(new_scale : Int) : Array(String)
       docker_api = Docr::API.new(Docr::Client.new)
-      names = [] of String
-      funko_containers = docker_api.containers.list(
-        all: true,
-      ).each { |container|
-        p! container.@names
-        container.@names.each { |name|
-          names << name.split("-", 2)[1].lstrip("/") if name.starts_with?("/faaso-")
+      current_scale = self.scale
+      result = [] of String
+
+      if current_scale == new_scale
+        Log.info { "Funko #{name} already at scale #{new_scale}" }
+        return result
+      end
+
+      Log.info { "Scaling #{name} from #{current_scale} to #{new_scale}" }
+      if new_scale > current_scale
+        (current_scale...new_scale).each {
+          Log.info { "Adding instance" }
+          result << (id = create_container)
+          start(id)
+          sleep 0.1.seconds
         }
+      else
+        # Sort them older to newer, so we stop the oldest
+        containers.sort! { |i, j|
+          i.@created <=> j.@created
+        }.each { |container|
+          Log.info { "Removing instance" }
+          docker_api.containers.stop(container.@id)
+          result << container.@id
+          current_scale -= 1
+          break if current_scale == new_scale
+          sleep 0.1.seconds
+        }
+      end
+
+      # And now, let's kill all the containers that are NOT running
+      containers.select { |container| container.@state != "running" }.each { |container|
+        Log.info { "Pruning dead instance" }
+        docker_api.containers.delete(container.@id)
       }
 
-      pp! names
-
-      from_names(names.to_a.sort!)
+      result
     end
 
     # Setup the target directory `path` with all the files needed
@@ -91,23 +132,18 @@ module Funko
     def prepare_build(path : Path)
       # Copy runtime if requested
       if !runtime.nil?
-        runtime_dir = Path.new("runtimes", runtime.as(String))
-        raise Exception.new("Error: runtime #{runtime} not found for funko #{name} in #{path}") unless File.exists?(runtime_dir)
-        Dir.glob("#{runtime_dir}/*").each { |src|
-          FileUtils.cp_r(src, path)
-        }
-        # Replace templates with processed files
-        context = _to_context
-        Dir.glob("#{path}/**/*.j2").each { |template|
-          dst = template[..-4]
-          File.open(dst, "w") do |file|
-            file << Crinja.render(File.read(template), context)
-          end
-          File.delete template
-        }
+        # Get runtime files list
+        runtime_base, runtime_files = Runtime.runtime_files(runtime.as(String))
+
+        Runtime.copy_templated(
+          runtime_base,
+          runtime_files,
+          path.to_s,
+          _to_context
+        )
       end
 
-      # Copy funko
+      # Copy funko on top of runtime
       raise Exception.new("Internal error: empty funko path for #{name}") if self.path.empty?
       Dir.glob("#{self.path}/*").each { |src|
         FileUtils.cp_r(src, path)
@@ -116,10 +152,22 @@ module Funko
 
     # Build image using docker in path previously prepared using `prepare_build`
     def build(path : Path)
+      Log.info { "Building image for #{name} in #{path}" }
       docker_api = Docr::API.new(Docr::Client.new)
+      tags = ["faaso-#{name}:latest"]
+      Log.info { "   Tags: #{tags}" }
       docker_api.images.build(
         context: path.to_s,
-        tags: ["faaso-#{name}:latest"]) { |x| Log.info { x } }
+        tags: tags,
+        no_cache: true) { |x| Log.info { x } }
+    end
+
+    def images
+      docker_api = Docr::API.new(Docr::Client.new)
+      docker_api.images.list.select { |image|
+        false if image.@repo_tags.nil?
+        true if image.@repo_tags.as(Array(String)).any?(&.starts_with?("faaso-#{name}:"))
+      }
     end
 
     # Return a list of image IDs for this funko, most recent first
@@ -135,76 +183,122 @@ module Funko
       end
     end
 
-    # Get all containers related to this funko
-    def containers
+    def latest_image
+      image_history.first
+    end
+
+    # Get all running containers related to this funko
+    def containers : Array(Docr::Types::ContainerSummary)
       docker_api = Docr::API.new(Docr::Client.new)
-      docker_api.containers.list(
-        all: true,
-        filters: {"name" => ["faaso-#{name}"]}
+      docker_api.containers.list(all: true).select { |container|
+        container.@names.any?(&.starts_with?("/faaso-#{name}-")) &&
+          container.@state == "running"
+      } || [] of Docr::Types::ContainerSummary
+    end
+
+    # A comprehensive status for the funko:
+    def docker_status
+      Status.new(
+        name: name,
+        containers: containers,
+        images: images,
+        scale: scale,
       )
     end
 
-    # Descriptive status for the funko
-    def status
-      status = self.containers.map { |container|
-        container.@status
-      }.join(", ")
-      status.empty? ? "Stopped" : status
-    end
-
-    # Is any instance of this funko running?
-    def running?
-      self.containers.any? { |container|
-        container.@state == "running"
-      }
-    end
-
-    # Is any instance of this funko paused?
-    def paused?
-      self.containers.any? { |container|
-        container.@state == "paused"
-      }
-    end
-
-    # Unpause paused container with the newer image
-    def unpause
+    # Start container with given id
+    def start(id : String)
       docker_api = Docr::API.new(Docr::Client.new)
-      images = self.image_history
-      paused = self.containers.select { |container|
-        container.@state == "paused"
-      }.sort! { |i, j|
-        (images.index(j.@image_id) || 9999) <=> (images.index(i.@image_id) || 9999)
-      }
-      docker_api.containers.unpause(paused[0].@id) unless paused.empty?
+      begin
+        docker_api.containers.start(id)
+      rescue ex : Docr::Errors::DockerAPIError
+        Log.error { "#{ex}" } unless ex.status_code == 304 # This just happens
+      end
     end
 
-    # Is any instance of this funko exited?
-    def exited?
-      self.containers.any? { |container|
-        container.@state == "exited"
-      }
-    end
-
-    # Restart exited container with the newer image
-    def start
-      # FIXME refactor DRY with unpause
+    # Wait up to `t` seconds for the funko to reach the desired scale
+    # If `healthy` is true, it will wait for the container to be declared
+    # healthy by the healthcheck
+    def wait_for(new_scale : Int, t : Int, healthy : Bool = false)
       docker_api = Docr::API.new(Docr::Client.new)
-      images = self.image_history
-      exited = self.containers.select { |container|
-        container.@state == "exited"
-      }.sort! { |i, j|
-        (images.index(j.@image_id) || 9999) <=> (images.index(i.@image_id) || 9999)
+      channel = Channel(Nil).new
+      spawn do
+        loop do
+          if healthy
+            channel.send(nil) if containers.select { |container|
+                                   begin
+                                     details = docker_api.containers.inspect(container.@id)
+                                     if details.nil?
+                                       false
+                                     elsif details.state.nil?
+                                       false
+                                     elsif details.state.as(Docr::Types::ContainerState).health.nil?
+                                       false
+                                     elsif details.state.as(Docr::Types::ContainerState).health.as(Docr::Types::Health).status == "healthy"
+                                       true
+                                     end
+                                     false
+                                   rescue ex : Docr::Errors::DockerAPIError
+                                     Log.error { "#{ex}" } unless ex.status_code == 304 # This just happens
+                                     false
+                                   end
+                                 } == new_scale
+          else
+            channel.send(nil) if scale == new_scale
+          end
+          sleep 0.2.seconds
+        end
+      end
+
+      select
+      when channel.receive
+        Log.info { "Funko #{name} reached scale #{new_scale}" }
+      when timeout(t.seconds)
+        raise Exception.new("Funko #{name} did not reach scale #{new_scale} in #{t} seconds")
+      end
+    end
+
+    # Remove all containers related to this funko
+    def remove_all_containers
+      docker_api = Docr::API.new(Docr::Client.new)
+      docker_api.containers.list(all: true).select { |container|
+        container.@names.any?(&.starts_with?("/faaso-#{name}-"))
+      }.each { |container|
+        begin
+          docker_api.containers.stop(container.@id) if container.status != "exited"
+        rescue ex : Docr::Errors::DockerAPIError
+          Log.error { "#{ex}" } unless ex.status_code == 304 # This just happens
+        end
+        docker_api.containers.delete(container.@id)
+      }
+    end
+
+    # Remove all images related to this funko
+    def remove_all_images
+      docker_api = Docr::API.new(Docr::Client.new)
+      docker_api.images.list.select { |image|
+        return false if image.@repo_tags.nil?
+        true if image.@repo_tags.as(Array(String)).any?(&.starts_with?("faaso-#{name}:"))
+      }.each { |image|
+        docker_api.images.delete(image.@id)
       }
-      docker_api.containers.restart(exited[0].@id) unless exited.empty?
     end
 
     # Create a container for this funko
     def create_container(autostart : Bool = true) : String
-      secrets_mount = "#{Dir.current}/secrets/#{name}"
+      # The path to secrets is tricky. On the server it will be in
+      # ./secrets/ BUT when you call on the Docker API you need to
+      # pass the path in the HOST SYSTEM WHERE DOCKER IS RUNNING
+      # so allow for a FAASO_SECRET_PATH override which will
+      # be set for the proxy container
+      secrets_mount = ENV.fetch(
+        "FAASO_SECRET_PATH",
+        "#{Dir.current}/secrets/#{name}"
+      )
       Dir.mkdir_p(secrets_mount)
       conf = Docr::Types::CreateContainerConfig.new(
         image: "faaso-#{name}:latest",
-        hostname: name,
+        hostname: "#{name}",
         # Port in the container side
         host_config: Docr::Types::HostConfig.new(
           network_mode: "faaso-net",
@@ -219,10 +313,43 @@ module Funko
       )
 
       docker_api = Docr::API.new(Docr::Client.new)
-      response = docker_api.containers.create(name: "faaso-#{name}", config: conf)
+      response = docker_api.containers.create(name: "faaso-#{name}-#{Random.base58(6)}", config: conf)
       response.@warnings.each { |msg| Log.warn { msg } }
       docker_api.containers.start(response.@id) if autostart
       response.@id
     end
+
+    # Create an array of funkos just from names. These are limited in function
+    # and can't call `prepare_build` or some other functionality
+    def self.from_names(names : Array(String)) : Array(Funko)
+      names.map { |name|
+        Funko.from_yaml("name: #{name}")
+      }
+    end
+
+    # Get all the funkos docker knows about.
+    def self.from_docker : Array(Funko)
+      docker_api = Docr::API.new(Docr::Client.new)
+      names = [] of String
+
+      # Get all containers that look like funkos
+      docker_api.containers.list(
+        all: true,
+      ).each { |container|
+        container.@names.each { |name|
+          names << name.split("-", 3)[1].lstrip("/") if name.starts_with?("/faaso-")
+        }
+      }
+
+      # Now get all images that look like funkos, since
+      # we can start them just fine.
+      docker_api.images.list.each { |image|
+        next if image.@repo_tags.nil?
+        image.@repo_tags.as(Array(String)).each { |tag|
+          names << tag.split("-", 2)[1].split(":", 2)[0] if tag.starts_with?("faaso-")
+        }
+      }
+      from_names(names.to_set.to_a.sort!)
+    end
   end
 end

src/main.cr

@@ -1,122 +1,62 @@
+require "./config.cr"
 require "./faaso.cr"
 require "colorize"
-require "commander"
+require "docopt"
+require "oplog"
+require "rucksack"
 
-# Log formatter for
-struct LogFormat < Log::StaticFormatter
-  @@colors = {
-    "FATAL" => :red,
-    "ERROR" => :red,
-    "WARN"  => :yellow,
-    "INFO"  => :green,
-    "DEBUG" => :blue,
-    "TRACE" => :light_blue,
-  }
-
-  def run
-    string "[#{Time.local}] #{@entry.severity.label}: #{@entry.message}".colorize(@@colors[@entry.severity.label])
-  end
-
-  def self.setup(quiet : Bool, verbosity)
-    if quiet
-      _verbosity = Log::Severity::Fatal
-    else
-      _verbosity = [
-        Log::Severity::Fatal,
-        Log::Severity::Error,
-        Log::Severity::Warn,
-        Log::Severity::Info,
-        Log::Severity::Debug,
-        Log::Severity::Trace,
-      ][[verbosity, 5].min]
-    end
-    Log.setup(
-      _verbosity,
-      Log::IOBackend.new(io: STDERR, formatter: LogFormat)
-    )
-  end
+macro version
+  "{{ `grep version shard.yml | cut -d: -f2` }}".strip()
 end
 
-cli = Commander::Command.new do |cmd|
-  cmd.use = "faaso"
-  cmd.long = "Functions as a Service, Open"
+doc = <<-DOC
+FaaSO CLI tool.
 
-  cmd.flags.add do |flag|
-    flag.name = "local"
-    flag.short = "-l"
-    flag.long = "--local"
-    flag.description = "Run commands locally instead of against a FaaSO server."
-    flag.default = false
-    flag.persistent = true
-  end
+Usage:
+  faaso build  FOLDER ...           [-v <level>] [-l] [--no-runtime]
+  faaso deploy FUNKO                [-v <level>] [-l]
+  faaso export SOURCE DESTINATION   [-v <level>]
+  faaso login                       [-v <level>]
+  faaso new -r runtime FOLDER       [-v <level>]
+  faaso scale FUNKO [SCALE]         [-v <level>] [-l]
+  faaso secret (-d|-a) FUNKO SECRET [-v <level>] [-l]
+  faaso status FUNKO                [-v <level>] [-l]
+  faaso version
+  faaso help COMMAND
 
-  cmd.flags.add do |flag|
-    flag.name = "quiet"
-    flag.short = "-q"
-    flag.long = "--quiet"
-    flag.description = "Don't log anything"
-    flag.default = false
-    flag.persistent = true
-  end
+Options:
+  -a --add         Add
+  -d --delete      Delete
+  -h --help        Show this screen
+  -l --local       Run commands locally instead of against a FaaSO server
+  --no-runtime     Don't merge a runtime into the funko
+  -r runtime       Runtime for the new funko (use -r list for examples)
+  -v level         Control the logging verbosity, 0 to 6 [default: 4]
+DOC
 
-  cmd.flags.add do |flag|
-    flag.name = "verbosity"
-    flag.short = "-v"
-    flag.long = "--verbosity"
-    flag.description = "Control the logging verbosity, 0 to 5 "
-    flag.default = 3
-    flag.persistent = true
-  end
+ans = Docopt.docopt(doc, ARGV)
+Oplog.setup(ans["-v"].to_s.to_i) unless ENV.fetch("FAASO_SERVER_SIDE", nil)
+Log.debug { ans }
 
-  cmd.commands.add do |command|
-    command.use = "build"
-    command.short = "Build a funko"
-    command.long = "Build a funko's Docker image and upload it to registry"
-    command.run do |options, arguments|
-      LogFormat.setup(options.@bool["quiet"], options.@int["verbosity"])
-      Faaso::Commands::Build.new(options, arguments).run
-    end
-  end
-
-  cmd.commands.add do |command|
-    command.use = "up"
-    command.short = "Ensure funkos are running"
-    command.long = "Start/unpause/create containers for requested funkos and ensure they are up."
-    command.run do |options, arguments|
-      LogFormat.setup(options.@bool["quiet"], options.@int["verbosity"])
-      Faaso::Commands::Up.new(options, arguments).run
-    end
-  end
-
-  cmd.commands.add do |command|
-    command.use = "deploy"
-    command.short = "Deploy latest images"
-    command.long = "Update containers for all funkos to latest image."
-    command.run do |options, arguments|
-      LogFormat.setup(options.@bool["quiet"], options.@int["verbosity"])
-      Faaso::Commands::Deploy.new(options, arguments).run
-    end
-  end
-
-  cmd.commands.add do |command|
-    command.use = "down"
-    command.short = "Stop a funko"
-    command.long = "Stop a funko in a container"
-    command.run do |options, arguments|
-      LogFormat.setup(options.@bool["quiet"], options.@int["verbosity"])
-      Faaso::Commands::Down.new(options, arguments).run
-    end
-  end
-
-  cmd.commands.add do |command|
-    command.use = "export"
-    command.short = "Export a funko to a directory"
-    command.long = "Exports a funko as a self-contained directory."
-    command.run do |options, arguments|
-      LogFormat.setup(options.@bool["quiet"], options.@int["verbosity"])
-      Faaso::Commands::Export.new(options, arguments).run
-    end
-  end
+case ans
+when .fetch("build", false)
+  exit Faaso::Commands::Build.new.run(ans, ans["FOLDER"].as(Array(String)))
+when .fetch("deploy", false)
+  exit Faaso::Commands::Deploy.new.run(ans, ans["FUNKO"].as(String))
+when .fetch("export", false)
+  exit Faaso::Commands::Export.new.run(ans, ans["SOURCE"].as(String), ans["DESTINATION"].as(String))
+when .fetch("login", false)
+  exit Faaso::Commands::Login.new.run(ans)
+when .fetch("new", false)
+  exit Faaso::Commands::New.new.run(ans, ans["FOLDER"].as(Array(String))[0])
+when .fetch("scale", false)
+  exit Faaso::Commands::Scale.new.run(ans, ans["FUNKO"].as(String), ans["SCALE"])
+when .fetch("secret", false)
+  exit Faaso::Commands::Secret.new.run(ans, ans["FUNKO"].as(String), ans["SECRET"].as(String))
+when .fetch("status", false)
+  exit Faaso::Commands::Status.new.run(ans, ans["FUNKO"].as(String))
+when .fetch("version", false)
+  Log.info { "#{version}" }
 end
 
-Commander.run(cli, ARGV)
+exit 0

src/runtime.cr

@@ -0,0 +1,104 @@
+require "rucksack"
+
+module Runtime
+  extend self
+
+  @@known : Array(String) = {{`find ./runtimes -type d -mindepth 1`.split('\n').reject(&.empty?)}}
+  @@filelist : Array(String) = {{`find ./runtimes -type f -mindepth 1`.split('\n').reject(&.empty?)}}
+
+  Log.debug { "@@known: #{@@known}" }
+  Log.debug { "@@filelist: #{@@filelist}" }
+
+  def list
+    Log.info { "Crystal has some included runtimes:\n" }
+    @@known.each do |i|
+      Log.info { "  * #{Path[i].basename}" }
+    end
+    Log.info { "\nOr if you have your own, use a folder name" }
+  end
+
+  def self.runtime_files(runtime : String) : {String, Array(String)}
+    runtime_base = ""
+    runtime_files = [] of String
+    if @@known.includes? "./runtimes/#{runtime}"
+      Log.info { "Using known runtime #{runtime}" }
+      runtime_base = "./runtimes/#{runtime}/"
+      runtime_files = @@filelist.select(&.starts_with?(runtime_base)).map { |path|
+        Path[path].normalize.to_s
+      }
+    elsif File.exists? runtime
+      Log.info { "Using directory #{runtime} as runtime" }
+      runtime_base = "#{runtime}"
+      runtime_files = Dir.glob("#{runtime_base}/**/*").select { |file| File.file?(file) }
+      runtime_files = runtime_files.map { |file| Path[file].normalize.to_s }
+    else
+      raise Exception.new("Can't find runtime #{runtime}")
+    end
+    {runtime_base, runtime_files.reject(&.starts_with? Path[runtime_base, "template"].normalize.to_s)}
+  end
+
+  def self.template_files(runtime : String) : {String, Array(String)}
+    template_base = ""
+    template_files = [] of String
+    if @@known.includes? "./runtimes/#{runtime}"
+      Log.info { "Using known runtime #{runtime}" }
+      template_base = "./runtimes/#{runtime}/template"
+      template_files = @@filelist.select(&.starts_with?(template_base))
+    elsif File.exists? runtime
+      Log.info { "Using directory #{runtime} as runtime" }
+      template_base = "#{runtime}/template"
+      template_files = Dir.glob("#{template_base}/**/*").select { |file| File.file?(file) }
+      template_files = template_files.map { |file| Path[file].normalize.to_s }
+    else
+      raise Exception.new("Can't find runtime #{runtime}")
+    end
+    {template_base, template_files}
+  end
+
+  # Copyes files from a runtime to a destination folder.
+  # Files ending in .j2 are rendered as Jinja2 templates
+  # using the provided context
+  def copy_templated(
+    base_path : String,
+    files : Array(String),
+    dst_path : String,
+    context
+  )
+    Dir.mkdir_p dst_path
+    files.each do |file|
+      content = IO::Memory.new
+      # We need to use RUCKSACK_MODE=0 so it
+      # fallbacks to the filesystem. Read the
+      # file into a buffer
+      rucksack(file).read(content)
+      if content.nil?
+        raise Exception.new("Can't find file #{file}")
+        return 1
+      end
+
+      # file is like "#{base}/foo"
+      # dst is like #{dst_path}/foo
+      dst = Path[dst_path] / Path[file].relative_to(base_path)
+      # Make sure we have dest dir
+      Dir.mkdir_p dst.dirname unless File.directory? dst.dirname
+      # Render templated files
+      if file.ends_with? ".j2"
+        dst = dst.sibling(dst.stem)
+        Log.info { "  Creating file #{dst} from #{file}" }
+        File.open(dst, "w") do |outf|
+          outf << Crinja.render(content.to_s, context)
+        end
+      else # Just copy the file
+        Log.info { "  Creating file #{dst} from #{file}" }
+        File.open(dst, "w") do |outf|
+          outf << content.to_s
+        end
+      end
+    end
+  end
+end
+
+# Embed runtimes in the faaso binary using rucksack
+{% for name in `find ./runtimes -type f`.split('\n') %}
+  rucksack({{name}})
+{% end %}

src/secrets.cr

@@ -15,9 +15,9 @@ module Secrets
       File.write(Path.new(funko_dir, name), value)
     end
     # Delete secrets not in the hash
-    Dir.glob(Path.new(SECRET_PATH, "*")).each do |funko_dir|
+    Dir.glob("#{SECRET_PATH}/*").each do |funko_dir|
       funko = File.basename(funko_dir)
-      Dir.glob(Path.new(funko_dir, "*")).each do |secret_file|
+      Dir.glob("#{funko_dir}/*").each do |secret_file|
         name = File.basename(secret_file)
         unless SECRETS.has_key?("#{funko}-#{name}")
           File.delete(secret_file)

src/views/funkos.ecr

@@ -1,26 +1,46 @@
 <%- result.each do |f| -%>
-<tr>
-  <td><%= f["name"] %></td>
-  <td><%= f["status"] %></td>
+<tr hx-indicator="#spinner-<%= f["name"] %>">
+  <td style="vertical-align: top;">
+    <%= f["name"] %>
+    <img id="spinner-<%= f["name"] %>" src="bars.svg"  class="htmx-indicator">
+  </td>
+  <td style="vertical-align: top;">
+    <%- f["containers"].as(Array(Docr::Types::ContainerSummary)).each do |c| -%>
+      <div class="grid">
+            <div>
+              <tt><%= c.@names[0].split("-")[-1] %></tt>
+            </div>
+            <div>
+              <%- if c.image_id == f["latest_image"] -%>
+                <span style="color:green;""> 🟢</span>
+              <%- else -%>
+                <span style="color:red;""> 🟢</span>
+              <%- end -%>
+            </div>
+            <div role="group">
+              <button hx-target="#terminal" hx-get="funkos/terminal/logs/<%= c.@names[0].lstrip("/") %>/">Logs</button>
+              <button hx-target="#terminal" hx-get="funkos/terminal/shell/<%= c.@names[0].lstrip("/") %>/">Shell</button>
+            </div>
+      </div>
+    <%- end -%>
+  </td>
+
   <td>
-  <%- if f["state"] == "running" -%>
-    <button state="disabled">Start</button>
-    <button>Pause</button>
-    <button>Stop</button>
-    <button>Restart</button>
-  <%- end -%>
-  <%- if f["state"] == "paused" -%>
-    <button>Start</button>
-    <button disabled>Pause</button>
-    <button>Stop</button>
-    <button>Restart</button>
-  <%- end -%>
-  <%- if f["state"] == "stopped" -%>
-    <button>Start</button>
-    <button disabled>Pause</button>
-    <button disabled>Stop</button>
-    <button disabled>Restart</button>
+  <div role="group">
+  <%- if f["name"] == "proxy" -%>
+  <%- else -%>
+    <%- if f["scale"].as(String).to_i > 0 -%>
+      <button disabled hx-get="funkos/<%= f["name"] %>/start">Start</button>
+      <button hx-get="funkos/<%= f["name"] %>/stop" hx-on:htmx:after-request="update_funkos()">Stop</button>
+      <button disabled hx-delete="funkos/<%= f["name"] %>/" hx-on:htmx:after-request="update_funkos()">Delete</button>
+    <%- else -%>
+      <button hx-get="funkos/<%= f["name"] %>/start" hx-on:htmx:after-request="update_funkos()">Start</button>
+      <button disabled hx-get="funkos/<%= f["name"] %>/stop" hx-on:htmx:after-request="update_funkos()">Stop</button>
+      <button hx-delete="funkos/<%= f["name"] %>/" hx-on:htmx:after-request="update_funkos()">Delete</button>
+    <%- end -%>
+      <button hx-get="funkos/<%= f["name"] %>/restart" hx-on:htmx:after-request="update_funkos()">Restart</button>
   <%- end -%>
+  </div>
   </td>
 </tr>
 <%- end -%>

src/views/secrets.ecr

@@ -0,0 +1,11 @@
+<%- result.each do |f| -%>
+<tr hx-indicator="#spinner-<%= f["name"] %>">
+  <td><%= f["funko"] %></td>
+  <td><%= f["name"] %></td>
+  <td>
+    <button hx-post="secrets/" hx-on:htmx:after-request="update_secrets()">Change</button>
+    <button hx-delete="secrets/<%= f["funko"] %>/<%= f["name"] %>/" hx-on:htmx:after-request="update_secrets()">Delete</button>
+    <img id="spinner-<%= f["name"] %>" src="bars.svg"  class="htmx-indicator">
+  </td>
+</tr>
+<%- end -%>

tinyproxy.conf

@@ -1,11 +0,0 @@
-
-  User nobody
-  Group nogroup
-  Port 8888
-  Listen 0.0.0.0
-  Timeout 600
-  Allow 0.0.0.0/0
-  ReverseOnly Yes
-  ReverseMagic Yes
-  ReversePath "/admin/" "http://127.0.0.1:3000/"
-  ReversePath "/faaso/hello/" "http://hello:3000/"