Added warmup service and ping service
This commit is contained in:
parent
5aab4f5ba9
commit
7fb5c54608
|
@ -15,6 +15,19 @@ else
|
||||||
make start_spamdetect &
|
make start_spamdetect &
|
||||||
echo "### Starting CoffeeHouse-CoreNLP ###"
|
echo "### Starting CoffeeHouse-CoreNLP ###"
|
||||||
make start_corenlp &
|
make start_corenlp &
|
||||||
|
echo "### Starting CoffeeHouse-NSFW ###"
|
||||||
|
make start_nsfw &
|
||||||
|
|
||||||
|
echo "### Checking Service Status ###"
|
||||||
|
|
||||||
|
if ! python3 scripts/warmup.py; then
|
||||||
|
echo "ERROR: Some services failed to start successfully duing the warmup process"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
echo "### Starting Ping Service ###"
|
||||||
|
make start_ping &
|
||||||
|
|
||||||
while sleep 60; do
|
while sleep 60; do
|
||||||
|
|
||||||
|
@ -35,11 +48,26 @@ else
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
ps aux | grep coffeehouse_nsfw | grep -q -v grep
|
||||||
|
NSFW_STATUS=$?
|
||||||
|
if [ $NSFW_STATUS -ne 0 ]; then
|
||||||
|
echo "ERROR: coffeehouse_nsfw has been terminated, terminating container."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ps aux | grep coffeehouse_ping | grep -q -v grep
|
||||||
|
PING_STATUS=$?
|
||||||
|
if [ $PING_STATUS -ne 0 ]; then
|
||||||
|
echo "ERROR: coffeehouse_ping has been terminated, terminating container."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
ps aux | grep edu.stanford.nlp.pipeline.StanfordCoreNLPServer | grep -q -v grep
|
ps aux | grep edu.stanford.nlp.pipeline.StanfordCoreNLPServer | grep -q -v grep
|
||||||
CORENLP_STATUS=$?
|
CORENLP_STATUS=$?
|
||||||
if [ $CORENLP_STATUS -ne 0 ]; then
|
if [ $CORENLP_STATUS -ne 0 ]; then
|
||||||
echo "ERROR: coffeehouse_corenlp has been terminated, terminating container."
|
echo "ERROR: coffeehouse_corenlp has been terminated, terminating container."
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
done
|
done
|
||||||
fi
|
fi
|
|
@ -25,4 +25,4 @@ echo " #### Preparing System for Java"
|
||||||
make system_prep_java
|
make system_prep_java
|
||||||
|
|
||||||
echo " #### Building and Installing CoffeeHousePy"
|
echo " #### Building and Installing CoffeeHousePy"
|
||||||
make install
|
make install_full
|
18
Makefile
18
Makefile
|
@ -28,6 +28,9 @@ clean_langdetect:
|
||||||
clean_spamdetect:
|
clean_spamdetect:
|
||||||
rm -rf services/spam_detection/build services/spam_detection/dist services/spam_detection/coffeehouse_spamdetection.egg-info
|
rm -rf services/spam_detection/build services/spam_detection/dist services/spam_detection/coffeehouse_spamdetection.egg-info
|
||||||
|
|
||||||
|
clean_ping:
|
||||||
|
rm -rf services/ping_service/build services/ping_service/dist services/ping_service/coffeehouse_ping.egg-info
|
||||||
|
|
||||||
clean_translation:
|
clean_translation:
|
||||||
rm -rf services/translation/build services/translation/dist services/translation/coffeehouse_translation.egg-info
|
rm -rf services/translation/build services/translation/dist services/translation/coffeehouse_translation.egg-info
|
||||||
|
|
||||||
|
@ -48,6 +51,7 @@ clean:
|
||||||
make clean_spamdetect
|
make clean_spamdetect
|
||||||
make clean_nsfw
|
make clean_nsfw
|
||||||
make clean_corenlp
|
make clean_corenlp
|
||||||
|
make clean_ping
|
||||||
|
|
||||||
# ======================================================================================================================
|
# ======================================================================================================================
|
||||||
|
|
||||||
|
@ -94,6 +98,9 @@ build_corenlp:
|
||||||
build_nsfw:
|
build_nsfw:
|
||||||
cd services/nsfw_detection; python3 setup.py build; python3 setup.py sdist
|
cd services/nsfw_detection; python3 setup.py build; python3 setup.py sdist
|
||||||
|
|
||||||
|
build_ping:
|
||||||
|
cd services/ping_service; python3 setup.py build; python3 setup.py sdist
|
||||||
|
|
||||||
build:
|
build:
|
||||||
make build_nlpfr
|
make build_nlpfr
|
||||||
make build_his
|
make build_his
|
||||||
|
@ -105,6 +112,7 @@ build:
|
||||||
make build_spamdetect
|
make build_spamdetect
|
||||||
make build_nsfw
|
make build_nsfw
|
||||||
make build_corenlp
|
make build_corenlp
|
||||||
|
make build_ping
|
||||||
|
|
||||||
# ======================================================================================================================
|
# ======================================================================================================================
|
||||||
|
|
||||||
|
@ -149,6 +157,9 @@ install_translation:
|
||||||
install_nsfw:
|
install_nsfw:
|
||||||
cd services/nsfw_detection; python3 setup.py install
|
cd services/nsfw_detection; python3 setup.py install
|
||||||
|
|
||||||
|
install_ping:
|
||||||
|
cd services/ping_service; python3 setup.py install
|
||||||
|
|
||||||
install_full:
|
install_full:
|
||||||
make install_rf
|
make install_rf
|
||||||
make install
|
make install
|
||||||
|
@ -163,6 +174,7 @@ install:
|
||||||
make install_spamdetect
|
make install_spamdetect
|
||||||
make install_nsfw
|
make install_nsfw
|
||||||
make build_corenlp
|
make build_corenlp
|
||||||
|
make build_ping
|
||||||
|
|
||||||
# ======================================================================================================================
|
# ======================================================================================================================
|
||||||
|
|
||||||
|
@ -199,6 +211,12 @@ start_nsfw:
|
||||||
start_corenlp:
|
start_corenlp:
|
||||||
cd services/corenlp; make start
|
cd services/corenlp; make start
|
||||||
|
|
||||||
|
start_ping:
|
||||||
|
python3 -m coffeehouse_ping --start-server
|
||||||
|
|
||||||
|
start_warmup:
|
||||||
|
cd scripts; python3 warmup.py
|
||||||
|
|
||||||
# ======================================================================================================================
|
# ======================================================================================================================
|
||||||
|
|
||||||
docker_build:
|
docker_build:
|
||||||
|
|
|
@ -39,6 +39,7 @@ make start_translate # Starts the translation server, runs on port 5603
|
||||||
|
|
||||||
| Name | Protocol | Port |
|
| Name | Protocol | Port |
|
||||||
|--------------------------------|----------|------|
|
|--------------------------------|----------|------|
|
||||||
|
| CoffeeHouse Ping Service | HTTP | 5600 |
|
||||||
| CoffeeHouse Spam Detection | HTTP | 5601 |
|
| CoffeeHouse Spam Detection | HTTP | 5601 |
|
||||||
| CoffeeHouse NSFW Classifier | HTTP | 5602 |
|
| CoffeeHouse NSFW Classifier | HTTP | 5602 |
|
||||||
| CoffeeHouse Translate | HTTP | 5603 |
|
| CoffeeHouse Translate | HTTP | 5603 |
|
||||||
|
|
Binary file not shown.
After Width: | Height: | Size: 84 KiB |
|
@ -0,0 +1,163 @@
|
||||||
|
"""
|
||||||
|
This script is executed on start-up after starting up the services, this is intended to check if all the services
|
||||||
|
has started correctly and load all the resources that are required are loaded, after this program is done
|
||||||
|
executing, the bootstrap should start the ping service so that programs know that CoffeeHouse-Utils is ready.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import urllib.parse
|
||||||
|
import requests
|
||||||
|
import base64
|
||||||
|
import time
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def core_nlp():
|
||||||
|
print("[CORENLP] Warming up")
|
||||||
|
|
||||||
|
current_tries = 0
|
||||||
|
max_tries = 30 # 5 Minutes
|
||||||
|
|
||||||
|
while True:
|
||||||
|
if current_tries > max_tries:
|
||||||
|
print("[CORENLP] ERROR! Too many attempts")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
r = requests.post("http://0.0.0.0:5604/?properties={\"annotators\": \"tokenize,ssplit,pos,ner,regexner,"
|
||||||
|
"sentiment\"}&pipelineLanguage=en",
|
||||||
|
|
||||||
|
headers={"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"},
|
||||||
|
data=urllib.parse.quote_plus("The quick brown fox jumped over the lazy dog")
|
||||||
|
)
|
||||||
|
if r.status_code == 200:
|
||||||
|
print("[CORENLP] OK")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
print("[CORENLP] Failed, Returned status code {0}, '{1}'".format(str(r.status_code), r.text))
|
||||||
|
except:
|
||||||
|
print("[CORENLP] Failed, request not completed")
|
||||||
|
pass
|
||||||
|
|
||||||
|
print("[CORENLP] Waiting 10 seconds")
|
||||||
|
time.sleep(10)
|
||||||
|
current_tries += 1
|
||||||
|
|
||||||
|
print("[CORENLP] Started successfully")
|
||||||
|
|
||||||
|
|
||||||
|
def langdetect():
|
||||||
|
print("[LANGDETECT] Warming up")
|
||||||
|
|
||||||
|
current_tries = 0
|
||||||
|
max_tries = 30 # 5 Minutes
|
||||||
|
while True:
|
||||||
|
if current_tries > max_tries:
|
||||||
|
print("[LANGDETECT] ERROR! Too many attempts")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
r = requests.post("http://0.0.0.0:5606/", data={"input": "The quick brown fox jumped over the lazy dog"})
|
||||||
|
if r.status_code == 200:
|
||||||
|
print("[LANGDETECT] OK")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
print("[LANGDETECT] Failed, Returned status code {0}, '{1}'".format(str(r.status_code), r.text))
|
||||||
|
except:
|
||||||
|
print("[LANGDETECT] Failed, request not completed")
|
||||||
|
pass
|
||||||
|
|
||||||
|
print("[LANGDETECT] Waiting 10 seconds")
|
||||||
|
time.sleep(10)
|
||||||
|
current_tries += 1
|
||||||
|
|
||||||
|
print("[LANGDETECT] Started successfully")
|
||||||
|
|
||||||
|
|
||||||
|
def spamdetect():
|
||||||
|
print("[SPAMDETECT] Warming up")
|
||||||
|
|
||||||
|
current_tries = 0
|
||||||
|
max_tries = 30 # 5 Minutes
|
||||||
|
while True:
|
||||||
|
if current_tries > max_tries:
|
||||||
|
print("[SPAMDETECT] ERROR! Too many attempts")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
r = requests.post("http://0.0.0.0:5601/", data={"input": "The quick brown fox jumped over the lazy dog"})
|
||||||
|
if r.status_code == 200:
|
||||||
|
print("[SPAMDETECT] OK")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
print("[SPAMDETECT] Failed, Returned status code {0}, '{1}'".format(str(r.status_code), r.text))
|
||||||
|
except:
|
||||||
|
print("[SPAMDETECT] Failed, request not completed")
|
||||||
|
pass
|
||||||
|
|
||||||
|
print("[SPAMDETECT] Waiting 10 seconds")
|
||||||
|
time.sleep(10)
|
||||||
|
current_tries += 1
|
||||||
|
|
||||||
|
print("[SPAMDETECT] Started successfully")
|
||||||
|
|
||||||
|
|
||||||
|
def nsfw():
|
||||||
|
print("[NSFW] Warming up")
|
||||||
|
|
||||||
|
current_tries = 0
|
||||||
|
max_tries = 30 # 5 Minutes
|
||||||
|
|
||||||
|
import os
|
||||||
|
print()
|
||||||
|
|
||||||
|
with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "red.jpg"), "rb") as img_file:
|
||||||
|
image_data = base64.b64encode(img_file.read()).decode('utf-8')
|
||||||
|
|
||||||
|
while True:
|
||||||
|
if current_tries > max_tries:
|
||||||
|
print("[NSFW] ERROR! Too many attempts")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
r = requests.post("http://0.0.0.0:5601/", data={
|
||||||
|
"input": image_data,
|
||||||
|
"type": "jpg"
|
||||||
|
})
|
||||||
|
if r.status_code == 200:
|
||||||
|
print("[NSFW] OK")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
print("[NSFW] Failed, Returned status code {0}, '{1}'".format(str(r.status_code), r.text))
|
||||||
|
except:
|
||||||
|
print("[NSFW] Failed, request not completed")
|
||||||
|
pass
|
||||||
|
|
||||||
|
print("[NSFW] Waiting 10 seconds")
|
||||||
|
time.sleep(10)
|
||||||
|
current_tries += 1
|
||||||
|
|
||||||
|
print("[NSFW] Started successfully")
|
||||||
|
|
||||||
|
|
||||||
|
print(" ( ) ( ) )")
|
||||||
|
print(" ) ( ) ( (")
|
||||||
|
print(" ( ) ( ) )")
|
||||||
|
print(" _____________")
|
||||||
|
print(" <_____________> ___")
|
||||||
|
print(" | |/ _ \\")
|
||||||
|
print(" | | | |")
|
||||||
|
print(" | |_| |")
|
||||||
|
print(" ___| |\___/")
|
||||||
|
print("/ \___________/ \\")
|
||||||
|
print("\_____________________/")
|
||||||
|
print()
|
||||||
|
print("CoffeeHouse-Utils Warmup")
|
||||||
|
|
||||||
|
core_nlp()
|
||||||
|
spamdetect()
|
||||||
|
langdetect()
|
||||||
|
nsfw()
|
||||||
|
|
||||||
|
print("OK, CoffeeHouse-Utils seems to be running fine.")
|
||||||
|
print("Exiting with code 0, the ping service should start in the next step.")
|
||||||
|
sys.exit(0)
|
|
@ -0,0 +1,4 @@
|
||||||
|
from . import server
|
||||||
|
from .server import *
|
||||||
|
|
||||||
|
__all__ = ["Server"]
|
|
@ -0,0 +1,50 @@
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from coffeehouse_ping import Server
|
||||||
|
|
||||||
|
|
||||||
|
def _real_main(argv=None):
|
||||||
|
"""
|
||||||
|
The main command-line processor
|
||||||
|
|
||||||
|
:param argv:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
if argv[1] == '--help':
|
||||||
|
_help_menu(argv)
|
||||||
|
if argv[1] == '--start-server':
|
||||||
|
_start_server(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def _start_server(argv=None):
|
||||||
|
"""
|
||||||
|
Starts the server
|
||||||
|
|
||||||
|
:param argv:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
server = Server()
|
||||||
|
server.start()
|
||||||
|
|
||||||
|
|
||||||
|
def _help_menu(argv=None):
|
||||||
|
"""
|
||||||
|
Displays the help menu and commandline usage
|
||||||
|
|
||||||
|
:param argv:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
print(
|
||||||
|
"CoffeeHouse Ping CLI\n\n"
|
||||||
|
" --help\n"
|
||||||
|
" --start-server\n"
|
||||||
|
)
|
||||||
|
sys.exit()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
try:
|
||||||
|
_real_main(sys.argv)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print('\nInterrupted by user')
|
|
@ -0,0 +1,44 @@
|
||||||
|
from hyper_internal_service import web
|
||||||
|
|
||||||
|
__all__ = ['Server']
|
||||||
|
|
||||||
|
|
||||||
|
class Server(object):
|
||||||
|
|
||||||
|
def __init__(self, port=5600):
|
||||||
|
"""
|
||||||
|
Public Constructor
|
||||||
|
:param port:
|
||||||
|
"""
|
||||||
|
self.port = port
|
||||||
|
self.web_application = web.Application()
|
||||||
|
self.web_application.add_routes(
|
||||||
|
[web.post('/', self.ping)]
|
||||||
|
)
|
||||||
|
|
||||||
|
async def ping(self, request):
|
||||||
|
"""
|
||||||
|
Handles the predict request "/", usage:
|
||||||
|
POST:: "input": str
|
||||||
|
:param request:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
post_data = await request.post()
|
||||||
|
return web.json_response({"status": True})
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
"""
|
||||||
|
Starts the web application
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
web.run_app(app=self.web_application, port=self.port)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""
|
||||||
|
Stops the web application
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
self.web_application.shutdown()
|
||||||
|
self.web_application.cleanup()
|
||||||
|
return True
|
|
@ -0,0 +1,16 @@
|
||||||
|
from setuptools import setup, find_packages
|
||||||
|
from setuptools.command.install import install
|
||||||
|
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name='coffeehouse_ping',
|
||||||
|
version='1.0.0',
|
||||||
|
description='Provides an endpoint for programs to ping for CoffeeHouse-Utils',
|
||||||
|
url='https://github.com/Intellivoid/CoffeeHousePy',
|
||||||
|
author='Zi Xing Narrakas',
|
||||||
|
author_email='netkas@intellivoid.info',
|
||||||
|
classifiers=[
|
||||||
|
'Development Status :: 3 - Internal/Alpha'
|
||||||
|
],
|
||||||
|
packages=find_packages()
|
||||||
|
)
|
Loading…
Reference in New Issue