From f4edb03979f06c6e534192f3acd98adf0d62d731 Mon Sep 17 00:00:00 2001 From: "Maxence G. de Montauzan" Date: Sat, 16 Dec 2023 03:01:38 +0100 Subject: [PATCH 1/6] Improve HTML template + improve links Reduce size (from ~15kB to ~4kB) Reduce incompatibility for emails Link to project on project name --- notifier.py | 17 ++++++++++---- template.html | 63 ++++++--------------------------------------------- 2 files changed, 20 insertions(+), 60 deletions(-) diff --git a/notifier.py b/notifier.py index d576fb0..2acc741 100644 --- a/notifier.py +++ b/notifier.py @@ -70,18 +70,18 @@ def main(): for new_r in new_releases: content += f""" -
  • {new_r["project_name"]} +
  • {get_html_project_link(new_r)} : new release - {new_r["release_tag"]} + {get_html_release_link(new_r)} available (old: {new_r["previous_tag"]}). (published {convert_date(new_r["published_date"])})
    • """ for new_p in new_projects: content += f""" -
  • {new_p["project_name"]} +
  • {get_html_project_link(new_p)} was added to your configuration. Last release: - {new_p["release_tag"]} + {get_html_release_link(new_p)} (published {convert_date(new_p["published_date"])})
    • """ with open(template_file, "r", encoding="utf-8") as f_template: @@ -114,6 +114,15 @@ def get_last_release(project): } +def get_html_project_link(el): + project_url = f'https://github.com/{el["project_name"]}' + return f'{el["project_name"]}' + + +def get_html_release_link(el): + return f'{el["release_tag"]}' + + def send_mail(content, config): smtp_port = config.get("smtp_port") smtp_server = config.get("smtp_server") diff --git a/template.html b/template.html index 60851ce..71a5ec5 100644 --- a/template.html +++ b/template.html @@ -5,62 +5,13 @@ - -

    - - Some new release on Github project available!

    -

    -
    - +
    +

    + + Some new release on Github project available!

    +

    +
    -- 2.49.1 From fb4ac8ea73621072580e2fb9692d726293fd085f Mon Sep 17 00:00:00 2001 From: "Maxence G. de Montauzan" Date: Sat, 30 Dec 2023 01:19:08 +0100 Subject: [PATCH 2/6] Justfile: improve release process & doc --- Justfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 609eeaf..b367a24 100644 --- a/Justfile +++ b/Justfile @@ -59,8 +59,9 @@ dpush: dbuild docker push {{ remote_build_image }} echo "To push a tagged version, do 'just release '" -# Release a version: create a tagged images, push it and create a git tag +# Use just a number! Without 'v'! Release a version - create a tagged images, push it and create git tag. release version: dbuild docker tag github-release-notifier {{ remote_image_name }}:{{ version }} docker push {{ remote_image_name }}:{{ version }} git tag -a v{{ version }} -m "" + git push --tags -- 2.49.1 From 9a8a7042b35ba67eb4b25faf610e96018716f9de Mon Sep 17 00:00:00 2001 From: "Maxence G. de Montauzan" Date: Sun, 12 Nov 2023 01:01:05 +0100 Subject: [PATCH 3/6] API to manage subscription Adapt docker part Improve log Add a just target --- Dockerfile | 5 +++- Justfile | 4 ++++ docker-compose.yml | 8 ++++--- notifier.py | 58 ++++++++++++++++++++++++++++++++++++++++------ requirements.txt | 6 +++-- 5 files changed, 68 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index e568243..ea1fe28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,13 @@ FROM python:3.10-alpine3.18 -RUN pip install requests WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir --upgrade --requirement requirements.txt COPY notifier.py template.html /app/ # TODO Dev purporse COPY conf.ini /app/conf.ini +EXPOSE 80 ENTRYPOINT ["python3", "/app/notifier.py"] +CMD ["uvicorn", "notifier:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/Justfile b/Justfile index b367a24..633cc68 100644 --- a/Justfile +++ b/Justfile @@ -12,6 +12,10 @@ remote_build_image := remote_image_name + ":" + last_commit_sha1 run: _ensure_venv_is_ok {{ python }} notifier.py +# Launch the API +api: _ensure_venv_is_ok + {{ venv }}/bin/uvicorn notifier:app --reload + # Init python virtual env init: python3 -m venv venv diff --git a/docker-compose.yml b/docker-compose.yml index 4eb8239..ac4399c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,13 +6,15 @@ services: image: github-release-notifier container_name: github-release-notifier volumes: - - ./conf.ini:/app/conf.ini + - ./conf.ini:/app/conf.ini + ports: + - 8000:80 mailpit: image: axllent/mailpit ports: - - "8025:8025" - - "1025:1025" + - 8025:8025 + - 1025:1025 ofelia: image: mcuadros/ofelia:latest diff --git a/notifier.py b/notifier.py index 2acc741..f4154dc 100644 --- a/notifier.py +++ b/notifier.py @@ -8,6 +8,11 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText import requests +from fastapi import FastAPI + +SCRIPT_DIR = os.path.dirname(__file__) +CONF_FILE = os.path.join(SCRIPT_DIR, "conf.ini") +TEMPLATE_FILE = os.path.join(SCRIPT_DIR, "template.html") class EnvInjection(configparser.Interpolation): @@ -26,15 +31,51 @@ class EnvInjection(configparser.Interpolation): return env_value if env_value else file_value -def main(): - script_dir = os.path.dirname(__file__) - conf_file = os.path.join(script_dir, "conf.ini") - template_file = os.path.join(script_dir, "template.html") +# +# API PARTS +# +app = FastAPI() + +@app.get("/subscriptions") +def get_projects(): + parser = load_conf() + projects = json.loads(parser["projects"].get("projects")) + return projects + + +@app.put("/subscriptions") +def put_projects(project: str, author: str | None = None): + # TODO Check if project really exist? + parser = load_conf() + projects = json.loads(parser.get("projects", "projects")) + + if author: + project = f"{author}/{project}" + projects.append(project) + + # TODO Watch a converter for list: https://stackoverflow.com/a/53274707/1346391 + parser.set("projects", "projects", json.dumps(projects, indent=0)) + + with open("conf.ini", "w", encoding="utf-8") as configfile: + parser.write(configfile) + + return project + + +def load_conf(conf_file=CONF_FILE) -> configparser.ConfigParser: parser = configparser.ConfigParser( default_section="config", interpolation=EnvInjection() ) parser.read(conf_file) + return parser + + +# +# SCRIPT PART +# +def main(): + parser = load_conf() default_config = parser["config"] try: @@ -67,8 +108,10 @@ def main(): return content = "" + news = [] for new_r in new_releases: + news.append(new_r["project_name"]) content += f"""
  • {get_html_project_link(new_r)} : new release @@ -77,6 +120,7 @@ def main(): (published {convert_date(new_r["published_date"])})
    • """ for new_p in new_projects: + news.append(new_p["project_name"]) content += f"""
  • {get_html_project_link(new_p)} was added to your configuration. @@ -84,10 +128,11 @@ def main(): {get_html_release_link(new_p)} (published {convert_date(new_p["published_date"])})
    • """ - with open(template_file, "r", encoding="utf-8") as f_template: + with open(TEMPLATE_FILE, "r", encoding="utf-8") as f_template: template = f_template.read() send_mail(template.replace("{{content}}", content), default_config) + print(f"Send a mail for new releases and projects on: {', '.join(news)}") with open("conf.ini", "w", encoding="utf-8") as configfile: parser.write(configfile) @@ -97,8 +142,7 @@ def get_last_release(project): url = f"https://api.github.com/repos/{project}/releases/latest" result = requests.get(url, timeout=10) - print(project) - print(url) + print(f"Check {project} - {url}") release = result.json() release_tag = release["tag_name"] published_date = release["published_at"] diff --git a/requirements.txt b/requirements.txt index 49977e2..fc62a3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ -requests -pre-commit black +fastapi +pre-commit +requests +uvicorn[standard] -- 2.49.1 From eea62d86852d4c97a798edf2f7faabfdedea0cdc Mon Sep 17 00:00:00 2001 From: "Maxence G. de Montauzan" Date: Sun, 12 Nov 2023 00:58:52 +0100 Subject: [PATCH 4/6] Use a configurable TOTP protection --- conf.ini | 4 ++++ notifier.py | 55 ++++++++++++++++++++++++++++++++++++++---------- requirements.txt | 1 + 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/conf.ini b/conf.ini index c0867e8..354a048 100644 --- a/conf.ini +++ b/conf.ini @@ -5,6 +5,10 @@ smtp_server = mailpit sender_email = sender@host.eu receiver_email = receiver@anotherhost.eu +[api.totp] +enabled = true +key = mysuperkey + [projects] projects = [ "borgbackup/borg", diff --git a/notifier.py b/notifier.py index f4154dc..bd7453d 100644 --- a/notifier.py +++ b/notifier.py @@ -7,9 +7,14 @@ import sys from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +import pyotp import requests -from fastapi import FastAPI +from fastapi import FastAPI, status +from fastapi.responses import JSONResponse +# +# CONF PART +# SCRIPT_DIR = os.path.dirname(__file__) CONF_FILE = os.path.join(SCRIPT_DIR, "conf.ini") TEMPLATE_FILE = os.path.join(SCRIPT_DIR, "template.html") @@ -31,10 +36,36 @@ class EnvInjection(configparser.Interpolation): return env_value if env_value else file_value +def load_conf(conf_file=CONF_FILE) -> configparser.ConfigParser: + parser = configparser.ConfigParser( + default_section="config", interpolation=EnvInjection() + ) + parser.read(conf_file) + return parser + + +def load_totp(): + parser = load_conf() + + if parser.getboolean("api.totp", "enabled", fallback=True): + totp = pyotp.TOTP(parser.get("api.totp", "key", fallback=pyotp.random_base32())) + # TODO Catch totp.now() error + print( + f"""TOTP enabled. + Information: {totp.provisioning_uri()} + TOTP check: {totp.now()}""" + ) + return totp + else: + print("WARNING: Api is open without security") + return type("totp", (object,), {"verify": lambda str: True}) + + # # API PARTS # app = FastAPI() +TOTP = load_totp() @app.get("/subscriptions") @@ -45,13 +76,23 @@ def get_projects(): @app.put("/subscriptions") -def put_projects(project: str, author: str | None = None): +def put_projects( + project: str, author: str | None = None, credentials: str | None = None +): + if not TOTP.verify(credentials): + return JSONResponse( + status_code=status.HTTP_401_UNAUTHORIZED, content="Unauthorized" + ) # TODO Check if project really exist? parser = load_conf() projects = json.loads(parser.get("projects", "projects")) if author: project = f"{author}/{project}" + + if project in projects: + return project + projects.append(project) # TODO Watch a converter for list: https://stackoverflow.com/a/53274707/1346391 @@ -60,15 +101,7 @@ def put_projects(project: str, author: str | None = None): with open("conf.ini", "w", encoding="utf-8") as configfile: parser.write(configfile) - return project - - -def load_conf(conf_file=CONF_FILE) -> configparser.ConfigParser: - parser = configparser.ConfigParser( - default_section="config", interpolation=EnvInjection() - ) - parser.read(conf_file) - return parser + return JSONResponse(status_code=status.HTTP_201_CREATED, content=project) # diff --git a/requirements.txt b/requirements.txt index fc62a3b..227cc3e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ black fastapi pre-commit +pyotp requests uvicorn[standard] -- 2.49.1 From 9d5e50c635986fa4976fde073088948fc8c6868d Mon Sep 17 00:00:00 2001 From: "Maxence G. de Montauzan" Date: Sun, 12 Nov 2023 00:42:05 +0100 Subject: [PATCH 5/6] Update README --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index eedd120..b90125c 100644 --- a/README.md +++ b/README.md @@ -12,33 +12,71 @@ Why using configuration?... ------------------------ ...instead use stared project of your Github account? Cause I've got about 80 "stared" projects and I don't wan't to be alerted for new releases of each of these project. -But, perhaps I'll add such a feature later on... And, Github API limits is at 60 requests by seconds, and I want to write this script really quickly in a first time. +But, perhaps I'll add such a feature later on... + How to use? ----------- Really simple! -* Edit conf.ini file to set `[config]` section +* Install dependances: `pip install -r requirements.txt` +* Edit conf.ini file to set `[config]` section: * your SMTP server configuration (host and port) * sender mail * receiver mail (:warning: not tested with more than 1 receiver) -* Add the projects you want to follow in the section `[project]` +* ...or use environment variable - same name but in upper case e.g. `SMTP_PORT` +* Add projects you want to follow in the section `[project]` * Be careful to follow a JSON valid syntax as in the provided file, i.e. coma after each `autor/project` except the last one. +Execute the script: `python notifier.py` + After first execution, the `conf.ini` file will be filled with last release tag found by the script, as you can see in the provided file. Hope you like, and have fun to read your mail! +Hey, what is this API? +-------------------------- +Since it's not really conveniant to go on a VM and edit the config.ini every time I find a cool new project whose new versions I want to keep track of, there is an API to do this. + +This is a really simple thing with no verification: you can add everything that you want! + +Just one endpoint: `subscription`, and two method: GET and PUT. +PUT take three parameters: +* `project`: full project name e.g. `MaxenceG2M/github-release-notifier` or "short" name (without author) e.g. `github-release-notifier` +* (optionnal) `author`: the author of the repository +* (optionnal) `credentials`: just the TOTP check code. + +Indeed, to have a minimal better-than-nothing protection, I opted for a simple TOTP code. No user/password or wathever. +You can disable-it in the `config.ini`, or specify a custom key. +If you don't specify a key, information will be displayed at startup (URL and the actual code. Be quick to check if it's ok!). + +To start the API: +```sh +$ uvicorn notifier:app +TOTP enabled. + Information: otpauth://totp/Secret?secret=mysuperkey + TOTP check: 377826 +INFO: Started server process [28264] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) +``` + +Cron-it with [ofelia](https://github.com/mcuadros/ofelia) +--------------------------------------------------------- +Ofelia is very conveniant for croning the verification part while letting the API run continuously. +So the `docker-compose.yml` use it to do this :) + +<3 + Problems I have to solve really quickly --------------------------------------- -I wrote this script really quickly, certainly faster than this README. So I already have two big proglems: -* The script sends mail even if no new projets or release has been detected -* The biggest, you have to edit script to specify absolute path... -* A lot of other little problems, like the code that's disgusting and so on. +One script to do all things. -For who's asking: yes, it's normal that I have put all code in one main function [like a blind gunner.](https://media.giphy.com/media/1yMexL5rkwYhuiVEmZ/giphy.gif). Really quickly I said! +For who's asking: yes, it's normal that I have put all code in ~~one main function~~ one main script [like a blind gunner.](https://media.giphy.com/media/1yMexL5rkwYhuiVEmZ/giphy.gif). +I want to keep this script as simple as possible. But overall, the script works and sends mail! -- 2.49.1 From 08d27d3183e80139d8a06716875d8f2efb23ffaf Mon Sep 17 00:00:00 2001 From: "Maxence G. de Montauzan" Date: Sun, 26 Nov 2023 00:32:51 +0100 Subject: [PATCH 6/6] Ofelia & Docker update --- Dockerfile | 1 - ofelia.ini | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ea1fe28..9158e76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,5 +9,4 @@ COPY notifier.py template.html /app/ COPY conf.ini /app/conf.ini EXPOSE 80 -ENTRYPOINT ["python3", "/app/notifier.py"] CMD ["uvicorn", "notifier:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/ofelia.ini b/ofelia.ini index 808e4a7..77f096a 100644 --- a/ofelia.ini +++ b/ofelia.ini @@ -4,6 +4,7 @@ smtp-port = 1025 email-to = max@ence.fr email-from = ofelia@container.sh -[job-run "notifier"] +[job-exec "notifier"] schedule = @every 30s container = github-release-notifier +command = python3 /app/notifier.py -- 2.49.1