Added warmup service and ping service

Docker/bootstrap

@@ -15,6 +15,19 @@ else
     make start_spamdetect &
     echo "### Starting CoffeeHouse-CoreNLP ###"
     make start_corenlp &
+    echo "### Starting CoffeeHouse-NSFW ###"
+    make start_nsfw &
+
+    echo "### Checking Service Status ###"
+
+    if ! python3 scripts/warmup.py; then
+      echo "ERROR: Some services failed to start successfully duing the warmup process"
+      exit 1
+    fi
+
+
+    echo "### Starting Ping Service ###"
+    make start_ping &
 
     while sleep 60; do
 
@@ -35,11 +48,26 @@ else
         exit 1
       fi
 
+      ps aux | grep coffeehouse_nsfw | grep -q -v grep
+      NSFW_STATUS=$?
+      if [ $NSFW_STATUS -ne 0 ]; then
+        echo "ERROR: coffeehouse_nsfw has been terminated, terminating container."
+        exit 1
+      fi
+
+      ps aux | grep coffeehouse_ping | grep -q -v grep
+      PING_STATUS=$?
+      if [ $PING_STATUS -ne 0 ]; then
+        echo "ERROR: coffeehouse_ping has been terminated, terminating container."
+        exit 1
+      fi
+
       ps aux | grep edu.stanford.nlp.pipeline.StanfordCoreNLPServer | grep -q -v grep
       CORENLP_STATUS=$?
       if [ $CORENLP_STATUS -ne 0 ]; then
         echo "ERROR: coffeehouse_corenlp has been terminated, terminating container."
         exit 1
       fi
+
     done
 fi

Docker/update_server

@@ -25,4 +25,4 @@ echo " #### Preparing System for Java"
 make system_prep_java
 
 echo " #### Building and Installing CoffeeHousePy"
-make install
+make install_full

Makefile

@@ -28,6 +28,9 @@ clean_langdetect:
 clean_spamdetect:
 	rm -rf services/spam_detection/build services/spam_detection/dist services/spam_detection/coffeehouse_spamdetection.egg-info
 
+clean_ping:
+	rm -rf services/ping_service/build services/ping_service/dist services/ping_service/coffeehouse_ping.egg-info
+
 clean_translation:
 	rm -rf services/translation/build services/translation/dist services/translation/coffeehouse_translation.egg-info
 
@@ -48,6 +51,7 @@ clean:
 	make clean_spamdetect
 	make clean_nsfw
 	make clean_corenlp
+	make clean_ping
 
 # ======================================================================================================================
 
@@ -94,6 +98,9 @@ build_corenlp:
 build_nsfw:
 	cd services/nsfw_detection; python3 setup.py build; python3 setup.py sdist
 
+build_ping:
+	cd services/ping_service; python3 setup.py build; python3 setup.py sdist
+
 build:
 	make build_nlpfr
 	make build_his
@@ -105,6 +112,7 @@ build:
 	make build_spamdetect
 	make build_nsfw
 	make build_corenlp
+	make build_ping
 
 # ======================================================================================================================
 
@@ -149,6 +157,9 @@ install_translation:
 install_nsfw:
 	cd services/nsfw_detection; python3 setup.py install
 
+install_ping:
+	cd services/ping_service; python3 setup.py install
+
 install_full:
 	make install_rf
 	make install
@@ -163,6 +174,7 @@ install:
 	make install_spamdetect
 	make install_nsfw
 	make build_corenlp
+	make build_ping
 
 # ======================================================================================================================
 
@@ -199,6 +211,12 @@ start_nsfw:
 start_corenlp:
 	cd services/corenlp; make start
 
+start_ping:
+	python3 -m coffeehouse_ping --start-server
+
+start_warmup:
+	cd scripts; python3 warmup.py
+
 # ======================================================================================================================
 
 docker_build:

README.md

@@ -39,6 +39,7 @@ make start_translate # Starts the translation server, runs on port 5603
 
 | Name                           | Protocol | Port |
 |--------------------------------|----------|------|
+| CoffeeHouse Ping Service       | HTTP     | 5600 |
 | CoffeeHouse Spam Detection     | HTTP     | 5601 |
 | CoffeeHouse NSFW Classifier    | HTTP     | 5602 |
 | CoffeeHouse Translate          | HTTP     | 5603 |

scripts/warmup.py

@@ -0,0 +1,163 @@
+"""
+This script is executed on start-up after starting up the services, this is intended to check if all the services
+has started correctly and load all the resources that are required are loaded, after this program is done
+executing, the bootstrap should start the ping service so that programs know that CoffeeHouse-Utils is ready.
+"""
+
+import urllib.parse
+import requests
+import base64
+import time
+import sys
+
+
+def core_nlp():
+    print("[CORENLP] Warming up")
+
+    current_tries = 0
+    max_tries = 30  # 5 Minutes
+
+    while True:
+        if current_tries > max_tries:
+            print("[CORENLP] ERROR! Too many attempts")
+            sys.exit(1)
+
+        try:
+            r = requests.post("http://0.0.0.0:5604/?properties={\"annotators\": \"tokenize,ssplit,pos,ner,regexner,"
+                              "sentiment\"}&pipelineLanguage=en",
+
+                              headers={"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"},
+                              data=urllib.parse.quote_plus("The quick brown fox jumped over the lazy dog")
+                              )
+            if r.status_code == 200:
+                print("[CORENLP] OK")
+                break
+            else:
+                print("[CORENLP] Failed, Returned status code {0}, '{1}'".format(str(r.status_code), r.text))
+        except:
+            print("[CORENLP] Failed, request not completed")
+            pass
+
+        print("[CORENLP] Waiting 10 seconds")
+        time.sleep(10)
+        current_tries += 1
+
+    print("[CORENLP] Started successfully")
+
+
+def langdetect():
+    print("[LANGDETECT] Warming up")
+
+    current_tries = 0
+    max_tries = 30  # 5 Minutes
+    while True:
+        if current_tries > max_tries:
+            print("[LANGDETECT] ERROR! Too many attempts")
+            sys.exit(1)
+
+        try:
+            r = requests.post("http://0.0.0.0:5606/", data={"input": "The quick brown fox jumped over the lazy dog"})
+            if r.status_code == 200:
+                print("[LANGDETECT] OK")
+                break
+            else:
+                print("[LANGDETECT] Failed, Returned status code {0}, '{1}'".format(str(r.status_code), r.text))
+        except:
+            print("[LANGDETECT] Failed, request not completed")
+            pass
+
+        print("[LANGDETECT] Waiting 10 seconds")
+        time.sleep(10)
+        current_tries += 1
+
+    print("[LANGDETECT] Started successfully")
+
+
+def spamdetect():
+    print("[SPAMDETECT] Warming up")
+
+    current_tries = 0
+    max_tries = 30  # 5 Minutes
+    while True:
+        if current_tries > max_tries:
+            print("[SPAMDETECT] ERROR! Too many attempts")
+            sys.exit(1)
+
+        try:
+            r = requests.post("http://0.0.0.0:5601/", data={"input": "The quick brown fox jumped over the lazy dog"})
+            if r.status_code == 200:
+                print("[SPAMDETECT] OK")
+                break
+            else:
+                print("[SPAMDETECT] Failed, Returned status code {0}, '{1}'".format(str(r.status_code), r.text))
+        except:
+            print("[SPAMDETECT] Failed, request not completed")
+            pass
+
+        print("[SPAMDETECT] Waiting 10 seconds")
+        time.sleep(10)
+        current_tries += 1
+
+    print("[SPAMDETECT] Started successfully")
+
+
+def nsfw():
+    print("[NSFW] Warming up")
+
+    current_tries = 0
+    max_tries = 30  # 5 Minutes
+
+    import os
+    print()
+
+    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "red.jpg"), "rb") as img_file:
+        image_data = base64.b64encode(img_file.read()).decode('utf-8')
+
+    while True:
+        if current_tries > max_tries:
+            print("[NSFW] ERROR! Too many attempts")
+            sys.exit(1)
+
+        try:
+            r = requests.post("http://0.0.0.0:5601/", data={
+                "input": image_data,
+                "type": "jpg"
+            })
+            if r.status_code == 200:
+                print("[NSFW] OK")
+                break
+            else:
+                print("[NSFW] Failed, Returned status code {0}, '{1}'".format(str(r.status_code), r.text))
+        except:
+            print("[NSFW] Failed, request not completed")
+            pass
+
+        print("[NSFW] Waiting 10 seconds")
+        time.sleep(10)
+        current_tries += 1
+
+    print("[NSFW] Started successfully")
+
+
+print("    (  )   (   )  )")
+print("     ) (   )  (  (")
+print("     ( )  (    ) )")
+print("     _____________")
+print("    <_____________> ___")
+print("    |             |/ _ \\")
+print("    |               | | |")
+print("    |               |_| |")
+print(" ___|             |\___/")
+print("/    \___________/    \\")
+print("\_____________________/")
+print()
+print("CoffeeHouse-Utils Warmup")
+
+core_nlp()
+spamdetect()
+langdetect()
+nsfw()
+
+print("OK, CoffeeHouse-Utils seems to be running fine.")
+print("Exiting with code 0, the ping service should start in the next step.")
+sys.exit(0)

services/ping_service/coffeehouse_ping/__init__.py

@@ -0,0 +1,4 @@
+from . import server
+from .server import *
+
+__all__ = ["Server"]

services/ping_service/coffeehouse_ping/__main__.py

@@ -0,0 +1,50 @@
+import sys
+
+from coffeehouse_ping import Server
+
+
+def _real_main(argv=None):
+    """
+    The main command-line processor
+
+    :param argv:
+    :return:
+    """
+    if argv[1] == '--help':
+        _help_menu(argv)
+    if argv[1] == '--start-server':
+        _start_server(argv)
+
+
+def _start_server(argv=None):
+    """
+    Starts the server
+
+    :param argv:
+    :return:
+    """
+    server = Server()
+    server.start()
+
+
+def _help_menu(argv=None):
+    """
+    Displays the help menu and commandline usage
+
+    :param argv:
+    :return:
+    """
+    print(
+        "CoffeeHouse Ping CLI\n\n"
+        "   --help\n"
+        "   --start-server\n"
+    )
+    sys.exit()
+
+
+
+if __name__ == '__main__':
+    try:
+        _real_main(sys.argv)
+    except KeyboardInterrupt:
+        print('\nInterrupted by user')

services/ping_service/coffeehouse_ping/server.py

@@ -0,0 +1,44 @@
+from hyper_internal_service import web
+
+__all__ = ['Server']
+
+
+class Server(object):
+
+    def __init__(self, port=5600):
+        """
+        Public Constructor
+        :param port:
+        """
+        self.port = port
+        self.web_application = web.Application()
+        self.web_application.add_routes(
+            [web.post('/', self.ping)]
+        )
+
+    async def ping(self, request):
+        """
+        Handles the predict request "/", usage:
+        POST:: "input": str
+        :param request:
+        :return:
+        """
+        post_data = await request.post()
+        return web.json_response({"status": True})
+
+    def start(self):
+        """
+        Starts the web application
+        :return:
+        """
+        web.run_app(app=self.web_application, port=self.port)
+        return True
+
+    def stop(self):
+        """
+        Stops the web application
+        :return:
+        """
+        self.web_application.shutdown()
+        self.web_application.cleanup()
+        return True

services/ping_service/setup.py

@@ -0,0 +1,16 @@
+from setuptools import setup, find_packages
+from setuptools.command.install import install
+
+
+setup(
+    name='coffeehouse_ping',
+    version='1.0.0',
+    description='Provides an endpoint for programs to ping for CoffeeHouse-Utils',
+    url='https://github.com/Intellivoid/CoffeeHousePy',
+    author='Zi Xing Narrakas',
+    author_email='netkas@intellivoid.info',
+    classifiers=[
+        'Development Status :: 3 - Internal/Alpha'
+    ],
+    packages=find_packages()
+)

test.py

@@ -0,0 +1,2 @@
+import os
+print(os.path.dirname(os.path.realpath(__file__)))