diff --git a/.gitignore b/.gitignore index 31a0bcc..696f579 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.faaso.yml # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/busqueda/README.md b/busqueda/README.md new file mode 100644 index 0000000..2518e6a --- /dev/null +++ b/busqueda/README.md @@ -0,0 +1,11 @@ +# Readme for Busqueda + +This is a funko using the Crystal runtime for [FaaSO](https://git.ralsina.me/ralsina/faaso) + +## What is Busqueda + +Write here what it is + +## How to use Busqueda + +And so on. \ No newline at end of file diff --git a/busqueda/__init__.py b/busqueda/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/busqueda/funko.cr b/busqueda/funko.cr new file mode 100644 index 0000000..e804231 --- /dev/null +++ b/busqueda/funko.cr @@ -0,0 +1,151 @@ +require "json" +require "kemal" +require "pg" +require "pool/connection" + +# get credentials from secrets +USER = File.read("/secrets/user").strip +PASS = File.read("/secrets/pass").strip +DBHOST = File.read("/secrets/dbhost").strip + +DBURL = "postgres://#{USER}:#{PASS}@#{DBHOST}:5432/nombres" +puts "Connnecting to #{DBURL}" + +# Create a connection pool to the database +pg = ConnectionPool.new(capacity: 5, timeout: 0.01.seconds) do + PG.connect(DBURL) +end + +def normalize(s : String) : String + s.unicode_normalize(:nfkd) + .chars.reject! { |character| + !character.ascii_letter? && (character != ' ') + }.join("").downcase +end + +# A basic hello world get endpoint +post "/" do |env| + prefijo = env.params.json["p"].as(String) + genero = env.params.json["g"].as(String) + year = env.params.json["a"].as(String) + + p! prefijo, genero, year + + prefijo = normalize(prefijo) + if !["f", "m"].includes?(genero) + genero = nil + end + datos = [] of Tuple(String, Int32 | String) + # Connect using credentials provided + pg.connection do |cursor| + if prefijo.empty? && year.empty? + result_set = cursor.query(" + SELECT nombre, total::integer + FROM totales + ORDER BY total DESC + LIMIT 50") + elsif prefijo.empty? && !year.empty? + # Per-year totals + result_set = cursor.query(" + SELECT nombre, contador::integer + FROM nombres + WHERE + anio = $1 + ORDER BY contador DESC + LIMIT 50", year) + elsif !prefijo.empty? && year.empty? + # Filter only by prefix + result_set = cursor.query(" + SELECT nombre, total::integer + FROM totales + WHERE + nombre LIKE $1 + ORDER BY total DESC + LIMIT 50", prefijo + "%") + elsif !prefijo.empty? && !year.empty? + # We have both + result_set = cursor.query(" + SELECT nombre, contador::integer + FROM nombres + WHERE + anio = $1 AND + nombre LIKE $2 + ORDER BY contador DESC + LIMIT 50", year, prefijo + "%") + end + + if !result_set.nil? + result_set.each do + nombre = result_set.read(String) + valor = result_set.read(Int32) + datos.push({nombre, valor}) + end + result_set.close + end + + if datos.empty? + raise "No data found" + end + end + # In this context, remove all composite names + datos.reject! { |row| + row[0].to_s.includes? " " + } + datos.insert(0, {"Nombre", "Cuantos?"}) + + if genero + pg.connection do |cursor| + datos.reject! { |row| + # How feminine is this name? + # Yes this database is upper case + puts "Checking #{row[1]} #{row[0]}" + feminidad = 0 + sql = %( + SELECT COALESCE((SELECT frecuencia FROM mujeres WHERE nombre='#{row[0]?.to_s.upcase}'), 0) AS mujeres, + COALESCE((SELECT frecuencia FROM hombres WHERE nombre='#{row[0]?.to_s.upcase}'), 0) AS hombres + ) + puts "SQL: #{sql}" + cursor.query sql do |result_set| + result_set.each do + mujeres = result_set.read(Int32) + hombres = result_set.read(Int32) + puts "frecuencias: #{mujeres} #{hombres}" + if hombres == mujeres == 0 + feminidad = 0.5 + else + feminidad = mujeres / (hombres + mujeres) + end + end + end + # El overlap en 0.5 es intencional! + if (feminidad >= 0.5 && genero == "f") || + (feminidad <= 0.5 && genero == "m") + false + else + true + end + } + puts "Data split by gender" + end + end + datos = datos[..10].map { |row| + [row[0].capitalize, row[1]] + } + + if datos.size > 1 + title = "¿Puede ser ... #{datos[0][1].to_s.titleize}? ¿O capaz que #{datos[1][1].to_s.titleize}? ¡Contame más!" + elsif datos.size == 1 + title = "Me parece que ... #{datos[0][1].to_s.titleize}!" + else + title = "No tengo idea!" + end + { + "title" => title, + "data" => datos, + }.to_json +end + +get "/ping/" do + pg.connection.exec("SELECT 42") + "OK" +end diff --git a/busqueda/funko.yml b/busqueda/funko.yml new file mode 100644 index 0000000..1e3e507 --- /dev/null +++ b/busqueda/funko.yml @@ -0,0 +1,11 @@ +name: busqueda +runtime: kemal +options: + shard_build_options: "--release" + ship_packages: [] + devel_packages: [] + healthcheck_options: "--interval=1m --timeout=2s --start-period=2s --retries=3" + healthcheck_command: "curl --fail http://localhost:3000/ping || exit 1" + copy_from_build: + - "public public" + - "bin/funko ." \ No newline at end of file diff --git a/busqueda/handler.py b/busqueda/handler.py deleted file mode 100644 index 2f4fe3f..0000000 --- a/busqueda/handler.py +++ /dev/null @@ -1,164 +0,0 @@ -import unicodedata -import urllib -from collections import namedtuple as nt -from dataclasses import dataclass -from json import loads -import logging - -import pygal -import pyrqlite.dbapi2 as dbapi2 -import requests - -connection = dbapi2.connect( - host="10.61.0.1", - user="root", - port=4001, - password="", -) - - -def remove_accents(input_str): - nfkd_form = unicodedata.normalize("NFKD", input_str) - return "".join([c for c in nfkd_form if not unicodedata.combining(c)]) - - -def femininidad(nombre): - sql1 = """ - SELECT COALESCE(frecuencia,0) - FROM mujeres WHERE nombre=:nombre - """ - sql2 = """ - SELECT COALESCE(frecuencia,0) - FROM hombres WHERE nombre=:nombre - """ - with connection.cursor() as cursor: - nombre = nombre.strip().split()[0].strip().upper() - mujeres = cursor.execute(sql1, {"nombre": nombre}).fetchone() - mujeres = 0 if mujeres is None else mujeres[0] - hombres = cursor.execute(sql2, {"nombre": nombre}).fetchone() - hombres = 0 if hombres is None else hombres[0] - if hombres == mujeres == 0: - return 0.5 - return mujeres / (hombres + mujeres) - - -def split_por_genero(nombres): - femeninos = [] - masculinos = [] - for n in nombres: - fem = femininidad(n[1]) - if fem is None: - femeninos.append(n) - masculinos.append(n) - elif fem >= 0.5: - femeninos.append(n) - else: - masculinos.append(n) - return {"f": femeninos, "m": masculinos} - - -def handle(req): - """handle a request to the function - Args: - req (str): request body - - { - p: prefijo del nombre, - g: genero del nombre, - y: año de nacimiento - } - - """ - - try: - data = loads(req) - except Exception as e: - data = {} - - try: - prefijo = data.get("p") or None - genero = data.get("g") or None - try: - año = int(data.get("a")) - except Exception: - año = None - except Exception as e: - prefijo = genero = año = None - return f"{req} -- {e}", 400 - - if prefijo is not None: - prefijo = prefijo.strip().lower() - - if genero not in ("f", "m"): - genero = None - - if prefijo is None and año is None: # Totales globales - with connection.cursor() as cursor: - sql = """ - SELECT total, nombre - FROM totales - ORDER BY total DESC - LIMIT 50 - """ - cursor.execute(sql) - datos = [(r["total"], r["nombre"]) for r in cursor.fetchall()] - - elif prefijo is None and año is not None: # Totales por año - with connection.cursor() as cursor: - sql = """ - SELECT contador, nombre - FROM nombres - WHERE - anio = :anio - ORDER BY contador DESC - LIMIT 50 - """ - cursor.execute(sql, {"anio": año}) - datos = [(r["contador"], r["nombre"]) for r in cursor.fetchall()] - - elif prefijo is not None and año is None: - with connection.cursor() as cursor: - sql = """ - SELECT total, nombre - FROM totales - WHERE - nombre LIKE :nombre - ORDER BY total DESC - LIMIT 50 - """ - cursor.execute(sql, {"nombre": f"{prefijo}%"}) - datos = [(r["total"], r["nombre"]) for r in cursor.fetchall()] - else: - with connection.cursor() as cursor: - sql = """ - SELECT contador, nombre - FROM nombres - WHERE - anio = :anio AND - nombre LIKE :nombre - ORDER BY contador DESC - LIMIT 50 - """ - cursor.execute(sql, {"anio": año, "nombre": f"{prefijo}%"}) - datos = [(r["contador"], r["nombre"]) for r in cursor.fetchall()] - - if genero: - datos = split_por_genero(datos)[genero] - - datos = datos[:10] - - chart = pygal.HorizontalBar(height=400, show_legend=False, show_y_labels=True) - chart.x_labels = [nombre.title() for _, nombre in datos[::-1]] - if len(datos) > 1: - chart.title = f"¿Puede ser ... {datos[0][1].title()}? ¿O capaz que {datos[1][1].title()}? ¡Contáme más!" - elif len(datos) == 1: - chart.title = f"¡Hola {datos[0][1].title()}!" - elif len(datos) < 1: - chart.title = "¡No esssistís!" - chart.add("", [contador for contador, _ in datos[::-1]]) - - return ( - chart.render(is_unicode=True), - 200, - {"Content-Type": "image/svg+xml"}, - ) diff --git a/busqueda/public/index.html b/busqueda/public/index.html new file mode 100644 index 0000000..f93807b --- /dev/null +++ b/busqueda/public/index.html @@ -0,0 +1,240 @@ + + + + + + + + + + + + + + + + + + + + + +
+
+

Adivinar nombres por edad y género

+
+
+
+
+ + + + +
+
+
+ + diff --git a/busqueda/requirements.txt b/busqueda/requirements.txt deleted file mode 100644 index cbc289f..0000000 --- a/busqueda/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pygal -requests -pyrqlite diff --git a/busqueda/shard.yml b/busqueda/shard.yml new file mode 100644 index 0000000..f610413 --- /dev/null +++ b/busqueda/shard.yml @@ -0,0 +1,18 @@ +name: busqueda +version: 0.1.0 + +targets: + funko: + main: main.cr + +dependencies: + kemal: + github: kemalcr/kemal + pg: + github: will/crystal-pg + pool: + github: ysbaddaden/pool + +# development_dependencies: +# webmock: +# github: manastech/webmock.cr diff --git a/c-historico/handler.cr b/c-historico/handler.cr deleted file mode 100644 index afe6419..0000000 --- a/c-historico/handler.cr +++ /dev/null @@ -1,127 +0,0 @@ -require "http/client" -require "http/headers" -require "http/request" -require "ishi/html" -require "json" -require "uuid" -require "db" -require "pg" - -USER = File.read("/var/openfaas/secrets/nombres-user").strip -PASS = File.read("/var/openfaas/secrets/nombres-pass").strip -DB_URL = "postgres://#{USER}:#{PASS}@10.61.0.1:5432/nombres" - -class Handler - def format_buffer(buffer, canvas_name, title = "") - # Process the gnuplot output so it works in the page - # - # buffer is the Ishi output - # name is a string to replace for gnuplot_canvas so - # we can have multiple charts in a page - # title is added on top of the chart - - html = buffer.to_s.split("\n") - html = html[html.index("")] - html = "#{title}" + html.join("\n") + %( -
- - - -
- - Sorry, your browser seems not to support the HTML 5 canvas element - -
- -
- ) - # This ID needs to be unique in case - # we have 2 charts in the same page - html.gsub("gnuplot_canvas", canvas_name) - end - - def query(sql, nombre) - # Runs a SQL query against the database. - # - # Returns an array of values [[Year,Count]...] - # Or nil if there are no results - - DB.open(DB_URL) do |cursor| - cursor.query(sql, nombre) do |result_set| - result = [] of Tuple(Int32, Int32) - result_set.each do - year = result_set.read(Int32) - contador = result_set.read(Int32) - result.push({year, contador}) - end - return result - end - end - # No result, return nil - nil - end - - nombres = [] of String - - def normalize_name(s) - # Remove diacritics, turn lowercase - normalized = s.unicode_normalize(:nfkd).chars - normalized.reject! { |character| - !character.ascii? - }.join("").downcase - end - - def run(request : HTTP::Request) - unless (body = request.body).nil? - query = JSON.parse(body) - nombres = query["i"].as_s.split(",").map(&.strip) - nombres.reject! { |nombre| nombre.size == 0 } - end - - if nombres.nil? || nombres.empty? - nombres = ["maria", "juan"] - end - - # Remove all diacritics and whatnot - nombres = nombres.map { |nombre| - normalize_name nombre - } - - puts "Processing #{nombres}" - - buffer = IO::Memory.new - Ishi.new(buffer) do - canvas_size(800, 300) - show_key(true) - xrange(1922..2015) - nombres.map { |nombre| - x = Array(Int32).new - y = Array(Int32).new - results = query("SELECT anio::integer, contador::integer FROM nombres WHERE nombre = $1 ORDER BY anio", nombre) - if results.nil? # No results, all 0s - x = (1922..2015).to_a - y = x.map { |_| 0 } - else # We got results - values = Hash(Int32, Int32).new(default_value: 0) - results.map { |row| - values[row[0]] = row[1] - } - (1922..2015).map { |year| - x << year - y << values[year] # Defaults to 0, so we have the whole set - } - end - plot(x, y, title: nombre.titleize, style: :lines, linewidth: 3) - } - end - puts "After Ishi" - - { - body: format_buffer(buffer, "historico"), - status_code: 200, - headers: HTTP::Headers{"Content-Type" => "text/html"}, - } - end -end diff --git a/c-historico/shard.yml b/c-historico/shard.yml deleted file mode 100644 index eb1bee6..0000000 --- a/c-historico/shard.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: historico -version: 0.1.0 - -dependencies: - ishi: - github: toddsundsted/ishi - pg: - github: will/crystal-pg diff --git a/deploy.sh b/deploy.sh index c1c97d3..8c6e1b3 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,5 +1,27 @@ #!/bin/sh -x set -e -export OPENFAAS_URL=http://pinky:8082 -pass faas.ralsina.me | faas-cli login -u admin --password-stdin -faas-cli deploy -f functions.yml $* +#export OPENFAAS_URL=http://pinky:8082 +#pass faas.ralsina.me | faas-cli login -u admin --password-stdin + +#pass iol-pass | faas-cli secret create iol-pass +#pass iol-user | faas-cli secret create iol-user +#pass iol-api-secret | faas-cli secret create iol-api-secret +#pass nombres-user | faas-cli secret create nombres-user +#pass nombres-pass | faas-cli secret create nombres-pass +#faas-cli deploy -f functions.yml $* + +export FAASO_SERVER=http://rocky:8888/admin +pass faaso-rocky | faaso login +pass nombres-user | faaso secret -a historico user +pass nombres-pass | faaso secret -a historico pass +echo "192.168.0.98" | faaso secret -a historico dbhost +faaso build busqueda +faaso scale busqueda 0 +faaso scale busqueda 1 +pass nombres-user | faaso secret -a busqueda user +pass nombres-pass | faaso secret -a busqueda pass +echo "192.168.0.98" | faaso secret -a busqueda dbhost +faaso build historico +faaso scale historico 0 +faaso scale historico 1 + diff --git a/functions.yml b/functions.yml index d35f3e0..b26b309 100644 --- a/functions.yml +++ b/functions.yml @@ -3,36 +3,10 @@ provider: name: openfaas gateway: http://pinky:8082 functions: - # busqueda: - # lang: python3-flask - # handler: ./busqueda - # image: ralsina/nombres_busqueda:latest - # historico: - # lang: python3-flask - # handler: ./historico - # image: ralsina/nombres_historico:latest tapas: lang: python3-flask handler: ./tapas image: ralsina/tapas:latest - c-historico: - lang: crystal-http - handler: ./c-historico - image: ralsina/c-historico:latest - build_args: - ADDITIONAL_PACKAGE: gnuplot - secrets: - - nombres-pass - - nombres-user - c-busqueda: - lang: crystal-http - handler: ./c-busqueda - image: ralsina/c-busqueda:latest - build_args: - ADDITIONAL_PACKAGE: gnuplot - secrets: - - nombres-pass - - nombres-user iol: lang: python3-fastapi handler: ./iol diff --git a/historico/README.md b/historico/README.md new file mode 100644 index 0000000..18ef2e5 --- /dev/null +++ b/historico/README.md @@ -0,0 +1,11 @@ +# Readme for Historico + +This is a funko using the Crystal runtime for [FaaSO](https://git.ralsina.me/ralsina/faaso) + +## What is Historico + +Write here what it is + +## How to use Historico + +And so on. diff --git a/historico/__init__.py b/historico/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/historico/funko.cr b/historico/funko.cr new file mode 100644 index 0000000..b1719b5 --- /dev/null +++ b/historico/funko.cr @@ -0,0 +1,66 @@ +require "json" +require "kemal" +require "pg" +require "pool/connection" + +# get credentials from secrets +USER = File.read("/secrets/user").strip +PASS = File.read("/secrets/pass").strip +DBHOST = File.read("/secrets/dbhost").strip + +DBURL = "postgres://#{USER}:#{PASS}@#{DBHOST}:5432/nombres" +puts "Connnecting to #{DBURL}" + +# Create a connection pool to the database +pg = ConnectionPool.new(capacity: 5, timeout: 0.01.seconds) do + PG.connect(DBURL) +end + +# Connect to the database and get information about +# the requested names +get "/" do |env| + # Names are query parameters + # Split by commas, capitalize and take the first 5 + names = env.params.query["names"] + .split(",").map(&.strip.capitalize)[..4] + + # Prepare results table + results = [] of Array(String) + results << ["Año"] + names + (1922..2015).each do |anio| + results << [anio.to_s] + end + # Connect using credentials provided + pg.connection do |cursor| + # Get the information for each name + names.map do |name| + # Normalize: remove diacritics etc. + name = name.unicode_normalize(:nfkd) + .chars.reject! { |character| + !character.ascii_letter? && (character != ' ') + }.join("").downcase + + counter_per_year = {} of Int32 => Int32 + cursor.query(" + SELECT anio::integer, contador::integer + FROM nombres WHERE nombre = $1", name) do |result_set| + result_set.each do + counter_per_year[result_set.read(Int32)] = result_set.read(Int32) + end + end + (1922..2015).each do |anio| + results[anio - 1921] << counter_per_year.fetch(anio, 0).to_s + end + end + end + results.to_json +end + +# The `/ping/` endpoint is configured in the container as a healthcheck +# You can make it better by checking that your database is responding +# or whatever checks you think are important +# +get "/ping/" do + pg.connection.exec("SELECT 42") + "OK" +end diff --git a/historico/funko.yml b/historico/funko.yml new file mode 100644 index 0000000..d5d9e85 --- /dev/null +++ b/historico/funko.yml @@ -0,0 +1,11 @@ +name: historico +runtime: kemal +options: + shard_build_options: "" + ship_packages: [] + devel_packages: [] + healthcheck_options: "--interval=1m --timeout=2s --start-period=2s --retries=3" + healthcheck_command: "curl --fail http://localhost:3000/ping || exit 1" + copy_from_build: + - "public public" + - "bin/funko ." diff --git a/historico/handler.py b/historico/handler.py deleted file mode 100644 index ca78b3a..0000000 --- a/historico/handler.py +++ /dev/null @@ -1,66 +0,0 @@ -import unicodedata -import urllib -from collections import defaultdict as ddict -from dataclasses import dataclass -from json import loads - -import pygal -import pyrqlite.dbapi2 as dbapi2 -import requests - -connection = dbapi2.connect( - host="10.61.0.1", - user="root", - port=4001, - password="", -) - - -def remove_accents(input_str): - nfkd_form = unicodedata.normalize("NFKD", input_str) - return "".join([c for c in nfkd_form if not unicodedata.combining(c)]) - - -def handle(req): - """handle a request to the function - Args: - req (str): request body - - {"i": ["nombre1, nombre2"]} - - """ - nombres = [] - try: - nombres = loads(req) - nombres = nombres["i"].split(",") - nombres = [remove_accents(x.strip().lower()) for x in nombres] - nombres = [n for n in nombres if n] - except Exception: - pass - - if not nombres: - nombres = ["maria", "juan"] - - chart = pygal.Line( - height=200, fill=True, human_readable=True, show_minor_x_labels=False - ) - chart.x_labels = [str(x) for x in range(1922, 2015)] - chart.x_labels_major = [str(x) if x % 10 == 0 else "" for x in range(1922, 2015)] - for nombre in nombres: - datos = ddict(int) - with connection.cursor() as cursor: - sql = """ - SELECT anio, contador, nombre - FROM nombres - WHERE nombre = :nombre - ORDER BY anio - """ - cursor.execute(sql, {"nombre": nombre}) - datos.update({r["anio"]: r["contador"] for r in cursor.fetchall()}) - chart.add(nombre.title(), [datos[x] for x in range(1922, 2015)]) - - chart.x_labels = [str(n) for n in range(1922, 2015)] - chart.x_labels_major = [str(n) for n in range(1920, 2020, 10)] - - # return Response(chart.render(is_unicode=True), mimetype="image/svg+xml") - return chart.render(is_unicode=True), 200, {"Content-Type": "image/svg+xml"} diff --git a/historico/public/index.html b/historico/public/index.html new file mode 100644 index 0000000..2f71980 --- /dev/null +++ b/historico/public/index.html @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + + + + + + + + +
+
+

Popularidad de Nombres en Argentina

+
+
+
+ + +
+
+ + diff --git a/historico/requirements.txt b/historico/requirements.txt deleted file mode 100644 index cbc289f..0000000 --- a/historico/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pygal -requests -pyrqlite diff --git a/historico/shard.yml b/historico/shard.yml new file mode 100644 index 0000000..49bf457 --- /dev/null +++ b/historico/shard.yml @@ -0,0 +1,14 @@ +name: historico +version: 0.1.0 + +targets: + funko: + main: main.cr + +dependencies: + kemal: + github: kemalcr/kemal + pg: + github: will/crystal-pg + pool: + github: ysbaddaden/pool diff --git a/c-busqueda/handler.cr b/old-busqueda/handler.cr similarity index 100% rename from c-busqueda/handler.cr rename to old-busqueda/handler.cr diff --git a/c-busqueda/shard.yml b/old-busqueda/shard.yml similarity index 100% rename from c-busqueda/shard.yml rename to old-busqueda/shard.yml