Ofelia & Docker update

Adapt docker part
Improve log
Add a just target

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Black format commit
+89e88c8a1b26e2c6ca242d0bba6cdba8da35c3ae

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+      - id: check-merge-conflict
+  - repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.1.1
+    hooks:
+      - id: autoflake
+  - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+    rev: v2.11.0
+    hooks:
+      - id: pretty-format-yaml
+        args: [--autofix, --indent, '2', --offset, '2']

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.10-alpine3.18
+
+WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir --upgrade --requirement requirements.txt
+COPY notifier.py template.html /app/
+
+# TODO Dev purporse
+COPY conf.ini /app/conf.ini
+
+EXPOSE 80
+CMD ["uvicorn", "notifier:app", "--host", "0.0.0.0", "--port", "80"]

Justfile

@@ -0,0 +1,71 @@
+# https://github.com/casey/just
+
+venv   := "./venv"
+pip    := venv / "bin/pip"
+python := venv / "bin/python"
+
+last_commit_sha1   := `git rev-parse --short HEAD`
+remote_image_name  := "gitea.gdemontauzan.fr/maxenceg2m/github-release-notifier"
+remote_build_image := remote_image_name + ":" + last_commit_sha1
+
+# Run the script
+run: _ensure_venv_is_ok
+    {{ python }} notifier.py
+
+# Launch the API
+api: _ensure_venv_is_ok
+    {{ venv }}/bin/uvicorn notifier:app --reload
+
+# Init python virtual env
+init:
+    python3 -m venv venv
+    {{ pip }} install --requirement requirements.txt
+    sha256sum requirements.txt > {{ venv }}/requirements.sha
+
+# Inspiration: https://github.com/behave/behave/blob/afb6b6716cd0f3e028829416475312db804a6aa9/justfile
+_ensure_venv_is_ok:
+    #!/usr/bin/env python3
+    from subprocess import run
+    from os import path
+    if run("sha256sum -c {{ venv }}/requirements.sha", shell=True).returncode != 0:
+        run("just init", shell=True)
+
+# Clean workspace - remove venv - and init
+reinit: hclean init
+
+# Remove virtual env (venv)
+hclean:
+    rm -fr venv
+
+# Run docker compose then show logs
+dup: dbuild
+    docker compose up -d
+    docker compose logs
+
+# Build with docker compose
+dbuild:
+    docker compose build
+
+# Down docker compose then build
+drebuild: ddown dbuild
+
+# Down docker compose
+ddown:
+    docker compose down
+
+# Docker build without cache
+dforce-build:
+    docker compose build --no-cache
+
+# Push a working images on registry, tagged with commit-sha1
+dpush: dbuild
+    docker tag github-release-notifier {{ remote_build_image }}
+    docker push {{ remote_build_image }}
+    echo "To push a tagged version, do 'just release <version>'"
+
+# Use just a number! Without 'v'! Release a version - create a tagged images, push it and create git tag.
+release version: dbuild
+    docker tag github-release-notifier {{ remote_image_name }}:{{ version }}
+    docker push {{ remote_image_name }}:{{ version }}
+    git tag -a v{{ version }} -m ""
+    git push --tags

README.md

@@ -12,31 +12,72 @@ Why using configuration?...
 ------------------------
 ...instead use stared project of your Github account?
 Cause I've got about 80 "stared" projects and I don't wan't to be alerted for new releases of each of these project.
-But, perhaps I'll add such a feature later on...
 
 And, Github API limits is at 60 requests by seconds, and I want to write this script really quickly in a first time.
 
+But, perhaps I'll add such a feature later on...
+
 How to use?
 -----------
 Really simple!
 
-* Edit conf.ini file to set `[config]` section
+* Install dependances: `pip install -r requirements.txt`
+* Edit conf.ini file to set `[config]` section:
   * your SMTP server configuration (host and port)
   * sender mail
   * receiver mail (:warning: not tested with more than 1 receiver)
-* Add the projects you want to follow in the section `[project]`
+* ...or use environment variable - same name but in upper case e.g. `SMTP_PORT`
+* Add projects you want to follow in the section `[project]`
   * Be careful to follow a JSON valid syntax as in the provided file, i.e. coma after each `autor/project` except the last one.
 
+Execute the script: `python notifier.py`
+
 After first execution, the `conf.ini` file will be filled with last release tag found by the script, as you can see in the provided file.
 
 Hope you like, and have fun to read your mail!
 
+Hey, what is this API?
+--------------------------
+Since it's not really conveniant to go on a VM and edit the config.ini every time I find a cool new project whose new versions I want to keep track of, there is an API to do this.
+
+This is a really simple thing with no verification: you can add everything that you want!
+
+Just one endpoint: `subscription`, and two method: GET and PUT.
+PUT take three parameters:
+* `project`: full project name e.g. `MaxenceG2M/github-release-notifier` or "short" name (without author) e.g. `github-release-notifier`
+* (optionnal) `author`: the author of the repository
+* (optionnal) `credentials`: just the TOTP check code.
+
+Indeed, to have a minimal better-than-nothing protection, I opted for a simple TOTP code. No user/password or wathever.
+You can disable-it in the `config.ini`, or specify a custom key.
+If you don't specify a key, information will be displayed at startup (URL and the actual code. Be quick to check if it's ok!).
+
+To start the API:
+```sh
+$ uvicorn notifier:app
+TOTP enabled.
+    Information: otpauth://totp/Secret?secret=mysuperkey
+    TOTP check: 377826
+INFO:     Started server process [28264]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+INFO:     Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)
+```
+
+Cron-it with [ofelia](https://github.com/mcuadros/ofelia)
+---------------------------------------------------------
+Ofelia is very conveniant for croning the verification part while letting the API run continuously.
+So the `docker-compose.yml` use it to do this :)
+
+<3
+
 Problems I have to solve really quickly
 ---------------------------------------
-I wrote this script really quickly, certainly faster than this README. So I already have two big proglems:
-* The script sends mail even if no new projets or release has been detected
-* The biggest, you have to edit script to specify absolute path...
-* A lot of other little problems, like the code that's disgusting and so on.
+One script to do all things.
+
+For who's asking: yes, it's normal that I have put all code in ~~one main function~~ one main script [like a blind gunner.](https://media.giphy.com/media/1yMexL5rkwYhuiVEmZ/giphy.gif).
+I want to keep this script as simple as possible.
+
 But overall, the script works and sends mail!
 
 Hey boy, what is the `pit.db` file?
@@ -44,5 +85,3 @@ Hey boy, what is the `pit.db` file?
 Oh, just for fun, and because I love this project, I use [pit by michaeldv](https://github.com/michaeldv/pit) to follow my task etc.
 
 It makes me think I should push my python version of this project on occasion when I will take the time to do...
-
-

conf.ini

@@ -1,10 +1,14 @@
 [config]
 github_api_url = https://api.github.com/repos/
-smtp_port = 25
-smtp_server = localhost
+smtp_port = 1025
+smtp_server = mailpit
 sender_email = sender@host.eu
 receiver_email = receiver@anotherhost.eu
 
+[api.totp]
+enabled = true
+key = mysuperkey
+
 [projects]
 projects = [
 	"borgbackup/borg",

docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3'
+
+services:
+  notifier:
+    build: .
+    image: github-release-notifier
+    container_name: github-release-notifier
+    volumes:
+      - ./conf.ini:/app/conf.ini
+    ports:
+      - 8000:80
+
+  mailpit:
+    image: axllent/mailpit
+    ports:
+      - 8025:8025
+      - 1025:1025
+
+  ofelia:
+    image: mcuadros/ofelia:latest
+    depends_on:
+      - notifier
+    command: daemon --config=/opt/config.ini
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./ofelia.ini:/opt/config.ini

notifier.py

@@ -1,103 +1,215 @@
-from configparser import ConfigParser
-import json
-import smtplib
-from email.mime.text import MIMEText
-from email.mime.multipart import MIMEMultipart
-
+import configparser
 import datetime
+import json
+import os
+import smtplib
+import sys
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 
+import pyotp
 import requests
+from fastapi import FastAPI, status
+from fastapi.responses import JSONResponse
 
-SMTP_PORT = 0
-SMTP_SERVER = 'null'
-SENDER_EMAIL = 'a@b.c'
-RECEIVER_EMAIL = 'd@e.f'
+#
+# CONF PART
+#
+SCRIPT_DIR = os.path.dirname(__file__)
+CONF_FILE = os.path.join(SCRIPT_DIR, "conf.ini")
+TEMPLATE_FILE = os.path.join(SCRIPT_DIR, "template.html")
 
+
+class EnvInjection(configparser.Interpolation):
+    """
+    Derived interpolation to take env variable before file variable.
+    Permit to keep the ini file for local / traditionnal use
+    And use env variable to overload configuration in a Docker usage.
+    """
+
+    def before_get(self, parser, section, option, value, defaults):
+        file_value = super().before_get(parser, section, option, value, defaults)
+        if section != parser.default_section:
+            return file_value
+
+        env_value = os.getenv(option.upper())
+        return env_value if env_value else file_value
+
+
+def load_conf(conf_file=CONF_FILE) -> configparser.ConfigParser:
+    parser = configparser.ConfigParser(
+        default_section="config", interpolation=EnvInjection()
+    )
+    parser.read(conf_file)
+    return parser
+
+
+def load_totp():
+    parser = load_conf()
+
+    if parser.getboolean("api.totp", "enabled", fallback=True):
+        totp = pyotp.TOTP(parser.get("api.totp", "key", fallback=pyotp.random_base32()))
+        # TODO Catch totp.now() error
+        print(
+            f"""TOTP enabled.
+    Information: {totp.provisioning_uri()}
+    TOTP check: {totp.now()}"""
+        )
+        return totp
+    else:
+        print("WARNING: Api is open without security")
+        return type("totp", (object,), {"verify": lambda str: True})
+
+
+#
+# API PARTS
+#
+app = FastAPI()
+TOTP = load_totp()
+
+
+@app.get("/subscriptions")
+def get_projects():
+    parser = load_conf()
+    projects = json.loads(parser["projects"].get("projects"))
+    return projects
+
+
+@app.put("/subscriptions")
+def put_projects(
+    project: str, author: str | None = None, credentials: str | None = None
+):
+    if not TOTP.verify(credentials):
+        return JSONResponse(
+            status_code=status.HTTP_401_UNAUTHORIZED, content="Unauthorized"
+        )
+    # TODO Check if project really exist?
+    parser = load_conf()
+    projects = json.loads(parser.get("projects", "projects"))
+
+    if author:
+        project = f"{author}/{project}"
+
+    if project in projects:
+        return project
+
+    projects.append(project)
+
+    # TODO Watch a converter for list: https://stackoverflow.com/a/53274707/1346391
+    parser.set("projects", "projects", json.dumps(projects, indent=0))
+
+    with open("conf.ini", "w", encoding="utf-8") as configfile:
+        parser.write(configfile)
+
+    return JSONResponse(status_code=status.HTTP_201_CREATED, content=project)
+
+
+#
+# SCRIPT PART
+#
 def main():
-    global SMTP_PORT, SMTP_SERVER, SENDER_EMAIL, RECEIVER_EMAIL
-    parser = ConfigParser()
-    parser.read('conf.ini')
+    parser = load_conf()
+    default_config = parser["config"]
 
-    SMTP_PORT = parser.get('config', 'smtp_port')
-    SMTP_SERVER = parser.get('config', 'smtp_server')
-    SENDER_EMAIL = parser.get('config', 'sender_email')
-    RECEIVER_EMAIL = parser.get('config', 'receiver_email')
+    try:
+        projects = json.loads(parser.get("projects", "projects"))
+    except json.decoder.JSONDecodeError as jse:
+        print("ERROR: config file is not correctly JSON formatted!", end="\n\t")
+        print(jse)
+        sys.exit(1)
 
-    projects = json.loads(parser.get('projects', 'projects'))
     new_releases = []
     new_projects = []
 
-    if not parser.has_section('release'):
-        parser.add_section('release')
+    if not parser.has_section("release"):
+        parser.add_section("release")
 
     for project in projects:
         last_release = get_last_release(project)
 
-        if not parser.has_option('release', project):
+        if not parser.has_option("release", project):
             new_projects.append(last_release)
         else:
-            last_config_tag = parser.get('release', project)
-            if last_config_tag != last_release['release_tag']:
-                last_release['preview_tag'] = last_config_tag
+            last_config_tag = parser.get("release", project)
+            if last_config_tag != last_release["release_tag"]:
+                last_release["previous_tag"] = last_config_tag
                 new_releases.append(last_release)
-        parser.set('release', project, last_release['release_tag'])
+        parser.set("release", project, last_release["release_tag"])
+
+    if not new_releases and not new_projects:
+        print("No new projets or new release detected. Bye!")
+        return
 
     content = ""
+    news = []
 
     for new_r in new_releases:
-        content += """
-        <li><a href="{}" target="_blank">{}</a>: new release <a href="{}" target="_blank">{}</a> available (old: {}).
-        (published {})</li>
-        """.format(
-            new_r['release_url'],
-            new_r['project_name'],
-            new_r['release_url'],
-            new_r['release_tag'],
-            new_r['preview_tag'],
-            convert_date(new_r['published_date']))
+        news.append(new_r["project_name"])
+        content += f"""
+        <li>{get_html_project_link(new_r)}
+        : new release
+        {get_html_release_link(new_r)}
+        available (old: {new_r["previous_tag"]}).
+        (published {convert_date(new_r["published_date"])})</li>"""
+
     for new_p in new_projects:
-        content += """
-        <li><a href="{}" target="_blank">{}</a> was added to your configuration. Last release: <a href="{}" target="_blank">{}</a>
-        (published {})</li>""".format(
-            new_p['release_url'],
-            new_p['project_name'],
-            new_p['release_url'],
-            new_p['release_tag'],
-            convert_date(new_p['published_date']))
+        news.append(new_p["project_name"])
+        content += f"""
+        <li>{get_html_project_link(new_p)}
+        was added to your configuration.
+        Last release:
+        {get_html_release_link(new_p)}
+        (published {convert_date(new_p["published_date"])})</li>"""
 
-    # print(content)
-
-    with open('template.html', 'r') as f_template:
+    with open(TEMPLATE_FILE, "r", encoding="utf-8") as f_template:
         template = f_template.read()
 
-    send_mail(template.replace('{{content}}', content))
+    send_mail(template.replace("{{content}}", content), default_config)
+    print(f"Send a mail for new releases and projects on: {', '.join(news)}")
 
-    with open('conf.ini', 'w') as configfile:
+    with open("conf.ini", "w", encoding="utf-8") as configfile:
         parser.write(configfile)
 
+
 def get_last_release(project):
-    url = 'https://api.github.com/repos/{}/releases/latest'.format(project)
-    result = requests.get(url)
+    url = f"https://api.github.com/repos/{project}/releases/latest"
+    result = requests.get(url, timeout=10)
 
-    print(project)
-    print(url)
+    print(f"Check {project} - {url}")
     release = result.json()
-    release_tag = release['tag_name']
-    published_date = release['published_at']
+    release_tag = release["tag_name"]
+    published_date = release["published_at"]
     # body = release['body']
-    release_url = release['html_url']
+    release_url = release["html_url"]
 
-    return {'release_tag': release_tag,
-            'published_date': published_date,
-            # 'body': body,
-            'project_name': project,
-            'release_url': release_url}
+    return {
+        "release_tag": release_tag,
+        "published_date": published_date,
+        # 'body': body,
+        "project_name": project,
+        "release_url": release_url,
+    }
+
+
+def get_html_project_link(el):
+    project_url = f'https://github.com/{el["project_name"]}'
+    return f'<a href="{project_url}" target="_blank">{el["project_name"]}</a>'
+
+
+def get_html_release_link(el):
+    return f'<a href="{el["release_url"]}" target="_blank">{el["release_tag"]}</a>'
+
+
+def send_mail(content, config):
+    smtp_port = config.get("smtp_port")
+    smtp_server = config.get("smtp_server")
+    sender_email = config.get("sender_email")
+    receiver_email = config.get("receiver_email")
 
-def send_mail(content):
     message = MIMEMultipart("alternative")
     message["Subject"] = "New Github releases"
-    message["From"] = SENDER_EMAIL
-    message["To"] = RECEIVER_EMAIL
+    message["From"] = sender_email
+    message["To"] = receiver_email
 
     # part1 = MIMEText(text, "plain")
     part2 = MIMEText(content, "html")
@@ -105,11 +217,13 @@ def send_mail(content):
     # message.attach(part1)
     message.attach(part2)
 
-    with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as server:
-        server.sendmail(SENDER_EMAIL, RECEIVER_EMAIL, message.as_string())
+    with smtplib.SMTP(smtp_server, smtp_port) as server:
+        server.sendmail(sender_email, receiver_email, message.as_string())
+
+
+def convert_date(date: str, dest_format="%d %b %Y at %H:%M") -> str:
+    return datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%SZ").strftime(dest_format)
 
-def convert_date(date: str, format='%d %b %Y at %H:%M') -> str:
-    return datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%SZ").strftime(format)
 
 if __name__ == "__main__":
     main()

ofelia.ini

@@ -0,0 +1,10 @@
+[global]
+smtp-host = mailpit
+smtp-port = 1025
+email-to = max@ence.fr
+email-from = ofelia@container.sh
+
+[job-exec "notifier"]
+schedule = @every 30s
+container = github-release-notifier
+command = python3 /app/notifier.py

requirements.txt

@@ -0,0 +1,6 @@
+black
+fastapi
+pre-commit
+pyotp
+requests
+uvicorn[standard]