From bd3642f0dea17ae20c04763c15d37e4a31b1fc93 Mon Sep 17 00:00:00 2001
From: Notoric <tom@acornes.com>
Date: Sun, 2 Jun 2024 00:24:46 +0100
Subject: [PATCH] Added docker build

---
 .gitignore                  |   3 +-
 Docker/Dockerfile           |   9 ++
 Docker/config_template.json |  20 +++
 Docker/docker-compose.yml   |  23 ++++
 Docker/generate-config.py   |  24 ++++
 Docker/requirements.txt     | Bin 0 -> 720 bytes
 Docker/snc.py               | 259 ++++++++++++++++++++++++++++++++++++
 script.py => snc.py         |   6 +-
 8 files changed, 341 insertions(+), 3 deletions(-)
 create mode 100644 Docker/Dockerfile
 create mode 100644 Docker/config_template.json
 create mode 100644 Docker/docker-compose.yml
 create mode 100644 Docker/generate-config.py
 create mode 100644 Docker/requirements.txt
 create mode 100644 Docker/snc.py
 rename script.py => snc.py (97%)

diff --git a/.gitignore b/.gitignore
index f9fab25..298087b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 config.json
-venv/*
\ No newline at end of file
+venv/*
+Docker/data/*
\ No newline at end of file
diff --git a/Docker/Dockerfile b/Docker/Dockerfile
new file mode 100644
index 0000000..dd4068b
--- /dev/null
+++ b/Docker/Dockerfile
@@ -0,0 +1,9 @@
+FROM python:3.9-slim
+WORKDIR /usr/src/app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD ["python", "-u", "snc.py"]
\ No newline at end of file
diff --git a/Docker/config_template.json b/Docker/config_template.json
new file mode 100644
index 0000000..c805c1a
--- /dev/null
+++ b/Docker/config_template.json
@@ -0,0 +1,20 @@
+{
+    "mongo" : {
+        "host" : "$MONGO_HOST",
+        "port" : "$MONGO_PORT",
+        "db" : "$MONGO_DB"
+    },
+    "groq" : {
+        "api_key" : "$GROQ_API_KEY"
+    },
+    "weather" : {
+        "api_key" : "$OPENWEATHERMAP_API_KEY",
+        "city" : "coalville,uk"
+    },
+    "news" : {
+        "api_key" : "$NEWSAPI_API_KEY",
+        "country" : "$NEWSAPI_COUNTRY",
+        "article_lifetime": "$NEWSAPI_ARTICLE_LIFETIME",
+        "article_interval": "$NEWSAPI_ARTICLE_INTERVAL"
+    }
+}
\ No newline at end of file
diff --git a/Docker/docker-compose.yml b/Docker/docker-compose.yml
new file mode 100644
index 0000000..d0a1c74
--- /dev/null
+++ b/Docker/docker-compose.yml
@@ -0,0 +1,23 @@
+version: '3'
+services:
+  python-script:
+    container_name: notoric-snc
+    build: .
+    environment:
+      - MONGO_HOST=mongodb
+      - MONGO_PORT=27017
+      - MONGO_DB=notoric-snc
+      - GROQ_API_KEY=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
+      - OPENWEATHERMAP_API_KEY=266adc688a1b7710b293f8b25de1bbe6
+      - OPENWEATHERMAP_CITY=London,uk
+      - NEWSAPI_API_KEY=188779213a2f4f6b87e3fbcdd906e8e0
+      - NEWSAPI_COUNTRY=gb
+      - ARTICLE_LIFETIME=6 # in hours
+      - ARTICLE_INTERVAL=1 # in hours
+  mongodb:
+    container_name: notoric-snc-mongo
+    image: mongo
+    ports:
+      - "27017:27017"
+    volumes:
+      - ./data:/data/db
\ No newline at end of file
diff --git a/Docker/generate-config.py b/Docker/generate-config.py
new file mode 100644
index 0000000..36cec46
--- /dev/null
+++ b/Docker/generate-config.py
@@ -0,0 +1,24 @@
+import os
+import json
+
+# Load the template
+with open('config_template.json', 'r') as template_file:
+    config_template = template_file.read()
+
+# Replace the placeholders with the actual values
+config = config_template.replace('$MONGO_HOST', os.environ['MONGO_HOST']) \
+    .replace('$MONGO_PORT', os.environ['MONGO_PORT']) \
+    .replace('$MONGO_DB', os.environ['MONGO_DB']) \
+    .replace('$GROQ_API_KEY', os.environ['GROQ_API_KEY']) \
+    .replace('$OPENWEATHERMAP_API_KEY', os.environ['OPENWEATHERMAP_API_KEY']) \
+    .replace('$OPENWEATHERMAP_CITY', os.environ['OPENWEATHERMAP_CITY']) \
+    .replace('$NEWSAPI_API_KEY', os.environ['NEWSAPI_API_KEY']) \
+    .replace('$NEWSAPI_COUNTRY', os.environ['NEWSAPI_COUNTRY']) \
+    .replace('$NEWSAPI_ARTICLE_LIFETIME', os.environ['ARTICLE_LIFETIME']) \
+    .replace('$NEWSAPI_ARTICLE_INTERVAL', os.environ['ARTICLE_INTERVAL'])
+
+# Write the config to a file
+with open('config.json', 'w') as config_file:
+    config_file.write(config)
+
+print("Config file created!")
\ No newline at end of file
diff --git a/Docker/requirements.txt b/Docker/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2e0139172d70b875e22b9247d2ebfdd093cdd974
GIT binary patch
literal 720
zcmZ9K(N2S44264b;-hRCpiVD*mSqACk_|T)3G?CA@3equNC*3SdU|^Pet+$);rXzu
z?TOL~Tk*cxneWbCc{aATEwQnCy93pbt*KD%fpbOVa%x(tYkMb@PsyZ?NI|8}kXr>y
z3wsH6<=tC5adxEY(>YX|QK5*cf?Nl_9XlsvUWn*iz|x`l2BHGdjNRAv1Savmr41&9
zXICi9!P$CF)K8338R*DeJR=95iRe{<NapioF@FW~6}B7&EuAJ-)jzt12{Fv|>B8F@
z_Zxherr&yvrYhWbIh%V{I&~-E&40x1kAj#<`m>qu<Q^SHFLi^zY*K!)o?rPLJ0OnC
z-3gA44i$(3+;K6faG|MV677LDA$CRn>ORBY*s<DgVF$xNuabYA<MoycQkE=krN`H!
N@5vinLPW=j^#_TMY03Zq

literal 0
HcmV?d00001

diff --git a/Docker/snc.py b/Docker/snc.py
new file mode 100644
index 0000000..753e379
--- /dev/null
+++ b/Docker/snc.py
@@ -0,0 +1,259 @@
+import time
+import os
+import json
+import pymongo
+import requests
+import schedule
+from groq import Groq
+from datetime import datetime, timedelta
+
+print("   _____ _   _  _____ ")
+print("  / ____| \ | |/ ____|")
+print(" | (___ |  \| | |")
+print("  \___ \|     | |")
+print("  ____) | |\  | |____")
+print(" |_____/|_| \_|\_____|")
+
+# Load config
+print("Loading config...")
+
+if os.path.exists('config.json') == False:
+    print("Config file not found, Creating...")
+    exec(open('generate-config.py').read())
+
+with open('config.json') as f:
+    config = json.load(f)
+
+print("Config loaded!")
+
+mongo_url = f"mongodb://{config['mongo']['host']}:{config['mongo']['port']}/"
+mongo_db = config['mongo']['db']
+
+weather_url = f"http://api.openweathermap.org/data/2.5/weather?q={config['weather']['city']}&appid={config['weather']['api_key']}&units=metric"
+
+news_url = f"http://newsapi.org/v2/top-headlines?country={config['news']['country']}&apiKey={config['news']['api_key']}"
+
+groq_key = config['groq']['api_key']
+
+# Connect to MongoDB
+print("Connecting to MongoDB...")
+
+client = pymongo.MongoClient(mongo_url)
+db = client[mongo_db]
+
+print("Connected to MongoDB!")
+
+# Create collections if they dont exist
+def create_collections():
+    collections = ['weather', 'newsfeed']
+    for collection in collections:
+        if collection not in db.list_collection_names():
+            db.create_collection(collection)
+            print(f"Created collection {collection}")
+
+# Get weather data
+
+def get_weather():
+    print("Getting weather data...")
+
+    response = requests.get(weather_url)
+    data = response.json()
+
+    response = {}
+
+    response['location'] = data['name']
+    temp = data['main']['temp']
+    temp = round(temp * 2 + 0.5) / 2
+    response['temperature'] = str(temp) + "°C"
+    response['humidity'] = str(data['main']['humidity']) + "%"
+    response['status'] = data['weather'][0]['description']
+    windspeed = data['wind']['speed']
+
+    if windspeed < 2:
+        response['wind'] = "Calm"
+    elif windspeed < 5:
+        response['wind'] = "Light Breeze"
+    elif windspeed < 11:
+        response['wind'] = "Gentle breeze"
+    elif windspeed < 17:
+        response['wind'] = "Moderate breeze"
+    elif windspeed < 23:
+        response['wind'] = "Strong breeze"
+    elif windspeed < 30:
+        response['wind'] = "High winds"
+    elif windspeed < 49:
+        response['wind'] = "Gale force winds"
+    else:
+        response['wind'] = "Storm"
+    
+    if data['visibility'] < 6000:
+        response['fog'] = "true"
+
+    print("Weather data retrieved!")
+    return response
+
+# Write weather data to MongoDB
+
+def write_weather():
+    print("Writing weather data to MongoDB...")
+
+    weather = get_weather()
+    weather['timestamp'] = datetime.now()
+
+    db.weather.replace_one({}, weather, upsert=True)
+
+    print("Weather data written to MongoDB!")
+
+# Get newsfeed data
+
+def get_newsfeed(category='general'):
+    print(f"Getting {category} newsfeed data...")
+
+    url = news_url + f"&category={category}"
+
+    response = requests.get(url)
+    data = response.json()
+
+    articles = []
+
+    for article in data['articles']:
+        article_data = {}
+        article_data['title'] = article['title']
+        article_data['url'] = article['url']
+        article_data['author'] = article['author']
+        article_data['category'] = category
+        article_data['timestamp'] = datetime.now()
+        articles.append(article_data)
+
+    print("Newsfeed data retrieved!")
+    return articles
+
+# Get most interesting news articles with AI
+
+def get_interesting_news(articles):
+    print("Getting interesting news...")
+    interesting_articles = []
+
+    try:
+        client = Groq(api_key=groq_key)
+        completion = client.chat.completions.create(
+            model="gemma-7b-it",
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You will be given an array of json elements, please provide the 3 indexes of the most interesting, important and notable news headlines that a mid-twenties person would like to read in the following format: {\"most_interesting\": {\"index\": index,\"title\": title},\"second_most_interesting\": {\"index\": index,\"title\": title},\"third_most_interesting\": {\"index\": index,\"title\": title}}"
+                },
+                {
+                    "role": "user",
+                    "content": str(articles)
+                }
+            ],
+            temperature=1.3,
+            max_tokens=1024,
+            top_p=1,
+            stream=False,
+            response_format={"type": "json_object"},
+            stop=None,
+        )
+
+        response = str(completion.choices[0].message.content)
+        response = response.replace("\n", " ")
+        response = json.loads(response)
+    except Exception as e: # If ai doesnt return a valid response, check anyway, if not use the first 3 articles
+        try:
+            response = e
+            response = response[18:]
+            response = json.loads(response)
+            response = response['error']['failed_generation']
+            response = response.replace("\n", " ")
+            response = json.loads(response)
+        except:
+            print("Error selecting articles! Using random selection...")
+            response = {
+                "most_interesting": {
+                    "index": 0,
+                    "title": "Interesting"
+                },
+                "second_most_interesting": {
+                    "index": 1,
+                    "title": "Interesting"
+                },
+                "third_most_interesting": {
+                    "index": 2,
+                    "title": "Interesting"
+                }
+            }
+
+    selected_articles = []
+
+    article_index = [0, 1, 2]
+    try:
+        article_index[0] = response['most_interesting']['index']
+        article_index[1] = response['second_most_interesting']['index']
+        article_index[2] = response['third_most_interesting']['index']
+        print("Selected articles:" + str(article_index))
+    except Exception as e:
+        print(e)
+        article_index = [0, 1, 2]
+        print("Using default article selection...")
+
+
+    for i in article_index:
+        article = articles[i]
+        selected_article = {}
+        selected_article['title'] = article['title']
+        selected_article['author'] = article['author']
+        selected_article['url'] = article['url']
+        selected_article['category'] = article['category']
+        selected_article['timestamp'] = datetime.now()
+        selected_articles.append(selected_article)
+
+    print("Interesting news retrieved!")
+
+    return selected_articles
+
+# Write newsfeed data to MongoDB
+
+def write_newsfeed(articles):
+    print("Writing newsfeed data to MongoDB...")
+
+    for article in articles:
+        db.newsfeed.replace_one({'url': article['url']}, article, upsert=True)
+
+    print("Newsfeed data written to MongoDB!")
+
+# Get articles from all newsfeeds
+
+def get_all_news():
+    print("Getting all news articles...")
+
+    write_newsfeed(get_interesting_news(get_newsfeed("technology")))
+    write_newsfeed(get_interesting_news(get_newsfeed("entertainment")))
+    write_newsfeed(get_interesting_news(get_newsfeed("science")))
+    
+# Delete all old news articles
+
+def delete_old_news():
+    print("Deleting old news articles...")
+
+    hrs = int(config['news']['article_lifetime'])
+
+    db.newsfeed.delete_many({'timestamp': {'$lt': datetime.now() - timedelta(hours=1) }})
+
+    print("Old news articles deleted!")
+
+# Main script
+
+create_collections()
+
+schedule.every(5).minutes.do(write_weather)
+schedule.every(int(config['news']['article_interval'])).hours.do(get_all_news)
+schedule.every(1).hours.do(delete_old_news)
+
+write_weather()
+get_all_news()
+delete_old_news()
+
+while True:
+    schedule.run_pending()
+    time.sleep(1)
\ No newline at end of file
diff --git a/script.py b/snc.py
similarity index 97%
rename from script.py
rename to snc.py
index 8da66a7..41f9051 100644
--- a/script.py
+++ b/snc.py
@@ -260,7 +260,9 @@ def get_all_news():
 def delete_old_news():
     print("Deleting old news articles...")
 
-    db.newsfeed.delete_many({'timestamp': {'$lt': datetime.now() - timedelta(hours=config['news']['article_lifetime']) }})
+    hrs = int(config['news']['article_lifetime'])
+
+    db.newsfeed.delete_many({'timestamp': {'$lt': datetime.now() - timedelta(hours=hrs) }})
 
     print("Old news articles deleted!")
 
@@ -269,7 +271,7 @@ def delete_old_news():
 create_collections()
 
 schedule.every(5).minutes.do(write_weather)
-schedule.every(config['news']['article_interval']).hours.do(get_all_news)
+schedule.every(int(config['news']['article_interval'])).hours.do(get_all_news)
 schedule.every(1).hours.do(delete_old_news)
 
 write_weather()