diff --git a/Makefile b/Makefile index a8c570cb..ef475ae9 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ clean_nlpfr: clean_alg: rm -rf alg/build alg/dist alg/coffeehouse_alg.egg-info +clean_rf: + rm -rf resource_fetch/build resource_fetch/dist resource_fetch/resource_fetch.egg-info + clean_his: rm -rf hyper_internal_service/build hyper_internal_service/dist hyper_internal_service/hyper_internal_service.egg-info @@ -24,68 +27,82 @@ clean: make clean_dltc make clean_his make clean_alg + make clean_rf # ====================================================================================================================== build_apt: - cd mods/apt; python3.6 setup.py build; python3.6 setup.py sdist + cd mods/apt; python3 setup.py build; python3 setup.py sdist build_stopwords: - cd mods/stopwords; python3.6 setup.py build; python3.6 setup.py sdist + cd mods/stopwords; python3 setup.py build; python3 setup.py sdist build_tokenizer: - cd mods/tokenizer; python3.6 setup.py build; python3.6 setup.py sdist + cd mods/tokenizer; python3 setup.py build; python3 setup.py sdist build_mods: make build_apt build_stopwords build_tokenizer build_nlpfr: make build_mods - cd nlpfr; python3.6 setup.py build; python3.6 setup.py sdist + cd nlpfr; python3 setup.py build; python3 setup.py sdist build_dltc: - cd dltc; python3.6 setup.py build; python3.6 setup.py sdist + cd dltc; python3 setup.py build; python3 setup.py sdist build_alg: - cd alg; python3.6 setup.py build; python3.6 setup.py sdist + cd alg; python3 setup.py build; python3 setup.py sdist build_his: - cd hyper_internal_service; python3.6 setup.py build; python3.6 setup.py sdist + cd hyper_internal_service; python3 setup.py build; python3 setup.py sdist + +build_rf: + cd resource_fetch; python3 setup.py build; python3 setup.py sdist build: make build_nlpfr make build_his make build_dltc make build_alg + make build_rf # ====================================================================================================================== +install_requirements: + python3 -m pip install -Ur requirements.txt + install_apt: - cd mods/apt; python3.6 setup.py install + cd mods/apt; python3 setup.py install install_stopwords: - cd mods/stopwords; python3.6 setup.py install + cd mods/stopwords; python3 setup.py install install_tokenizer: - cd mods/tokenizer; python3.6 setup.py install + cd mods/tokenizer; python3 setup.py install install_mods: make install_apt install_stopwords install_tokenizer install_nlpfr: make install_mods - cd nlpfr; python3.6 setup.py install + cd nlpfr; python3 setup.py install install_dltc: - cd dltc; python3.6 setup.py install + cd dltc; python3 setup.py install install_alg: - cd alg; python3.6 setup.py install + cd alg; python3 setup.py install install_his: - cd hyper_internal_service; python3.6 -m pip install -Ur dev_requirements.txt; python3.6 setup.py install + cd hyper_internal_service; python3 -m pip install -Ur dev_requirements.txt; python3 setup.py install + +install_rf: + # Make sure to run 'python3 -m resource_fetch ""' before using resource_fetch! + cd resource_fetch; python3 setup.py install install: + make install_requirements + make install_rf make install_nlpfr make install_his make install_dltc @@ -94,13 +111,13 @@ install: # ====================================================================================================================== system_prep_python: - apt -y install python3.6 python3-distutils python3-dev python3-setuptools + apt -y install python3 python3-distutils python3-dev python3-setuptools system_prep_pip: apt -y install wget curl wget https://bootstrap.pypa.io/get-pip.py - python3.6 get-pip.py + python3 get-pip.py rm get-pip.py system_prep_gcc: - apt -y install gcc build-essential + apt -y install gcc build-essential \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..fee5a5d1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,24 @@ +setuptools~=50.3.2 +requests~=2.22.0 +Keras~=2.2.5 +numpy~=1.19.4 +nltk~=3.5 +six~=1.15.0 +gensim~=3.8.3 +scikit-learn~=0.22.0 +nose~=1.3.7 +scipy~=1.3.3 +Click~=7.0 +tqdm~=4.51.0 +regex~=2020.11.11 +pyparsing~=2.4.6 +Unidecode~=1.1.1 +packaging~=20.3 +PyGithub~=1.54.1 +multidict~=4.7.6 +yarl~=1.4.2 +attrs~=19.3.0 +gunicorn~=20.0.4 +aiodns~=2.0.0 +cchardet~=2.1.6 +pytest~=5.4.2 \ No newline at end of file diff --git a/resource_fetch/README.md b/resource_fetch/README.md new file mode 100644 index 00000000..a7b1f552 --- /dev/null +++ b/resource_fetch/README.md @@ -0,0 +1,6 @@ +# Resource Fetch + +This CoffeeHouse library will allow you to set a Github Personal Access Token and use it +to fetch releases and extract them onto the system so that it can later be fetched within +runtime. This library will also keep track of the current version of the resource and update +it when there's a new version available. \ No newline at end of file diff --git a/resource_fetch/resource_fetch/__init__.py b/resource_fetch/resource_fetch/__init__.py new file mode 100644 index 00000000..a7e9aa0e --- /dev/null +++ b/resource_fetch/resource_fetch/__init__.py @@ -0,0 +1,2 @@ +from . import resource_fetch +from resource_fetch.resource_fetch import ResourceFetch diff --git a/resource_fetch/resource_fetch/__main__.py b/resource_fetch/resource_fetch/__main__.py new file mode 100644 index 00000000..ec769743 --- /dev/null +++ b/resource_fetch/resource_fetch/__main__.py @@ -0,0 +1,16 @@ +import sys + +from resource_fetch import ResourceFetch + + +def main(): + resource_fetcher = ResourceFetch() + if len(sys.argv[1:]) > 0: + resource_fetcher.define_pat(sys.argv[1]) + else: + resource_fetcher.define_pat(input("Personal Access Token: ")) + print("The Personal Access Token has been successfully defined") + + +if __name__ == '__main__': + main() diff --git a/resource_fetch/resource_fetch/resource_fetch.py b/resource_fetch/resource_fetch/resource_fetch.py new file mode 100644 index 00000000..eea96766 --- /dev/null +++ b/resource_fetch/resource_fetch/resource_fetch.py @@ -0,0 +1,186 @@ +import os +import sys +import tarfile +import zipfile +from pathlib import Path +from packaging import version +from github import Github +from urllib.request import urlretrieve +from urllib.parse import urlparse + + +class ResourceFetch(object): + + def __init__(self): + """ + ResourceFetch constructor + """ + + self.working_directory = os.path.join("/", "var", "resources") + self.pat_path = os.path.join(self.working_directory, ".pat") + self.github_client = None + if not os.path.exists(self.working_directory): + os.makedirs(self.working_directory) + + def rm(self, path): + """ + Recursively removes a directory or file + + :param path: + :return: + """ + + if os.path.isfile(path): + os.remove(path) + return + + directory = Path(path) + for item in directory.iterdir(): + if item.is_dir(): + self.rm(item) + else: + item.unlink() + directory.rmdir() + + def get_github(self): + if self.github_client is None: + self.github_client = Github(self.get_pat()) + return self.github_client + + def define_pat(self, token): + """ + Defines/overwrites the personal access token for fetching resources + + :param token: + :return: + """ + + if os.path.exists(self.pat_path): + self.rm(self.pat_path) + with open(self.pat_path, 'w+') as out: + out.write(token) + + def get_pat(self): + """ + Returns the personal access token that has been defined for fetching resources + + :return: + """ + if not os.path.exists(self.pat_path): + raise PermissionError("No Personal Access Token has been defined for ResourceFetcher") + with open(self.pat_path, 'r') as file: + return file.read() + + def fetch(self, organization, repository, check_for_update=True): + """ + Fetches a resource, downloads it if it's missing and or updates it if it's outdated + + :param organization: + :param repository: + :param check_for_update: + :return: + """ + resource_path = os.path.join(self.working_directory, repository) + + if os.path.exists(resource_path): + # The resource already exists, verify it + + if not check_for_update: # Skip the check for updates + return resource_path + + version_file = os.path.join(resource_path, ".version") + with open(version_file, 'r') as file: + current_version = file.read() + + # Check if it's outdated + latest_version = self.get_latest_version(organization, repository) + if version.parse(current_version) < version.parse(latest_version): + self.install_resource(organization, repository, resource_path) + return resource_path + self.install_resource(organization, repository, resource_path) + return resource_path + + def get_latest_version(self, organization, repository): + """ + Returns the latest version of the resource + + :param organization: + :param repository: + :return: + """ + releases = self.get_github().get_user(organization).get_repo(repository).get_releases() + return releases[0].title + + def get_latest_download_url(self, organization, repository): + """ + Gets the latest download URL for the requested resource + + :param organization: + :param repository: + :return: + """ + + releases = self.get_github().get_user(organization).get_repo(repository).get_releases() + return releases[0].get_assets()[0].browser_download_url + + @staticmethod + def reporthook(blocknum, blocksize, totalsize): + readsofar = blocknum * blocksize + if totalsize > 0: + percent = readsofar * 1e2 / totalsize + s = "\r%5.1f%% %*d / %d" % ( + percent, len(str(totalsize)), readsofar, totalsize) + sys.stderr.write(s) + if readsofar >= totalsize: # near the end + sys.stderr.write("\n") + else: # total size is unknown + sys.stderr.write("read %d\n" % (readsofar,)) + + def install_resource(self, organization, repository, path): + print("Installing {0} from {1}".format(repository, organization)) + temporary_directory = os.path.join(self.working_directory, "tmp") + if not os.path.exists(temporary_directory): + os.makedirs(temporary_directory) + + # Prepare the download + download_url = self.get_latest_download_url(organization, repository) + parsed_url = urlparse(download_url) + file_name = os.path.basename(parsed_url.path) + file_path = os.path.join(temporary_directory, file_name) + file_extension = os.path.splitext(file_name)[1] + + if os.path.exists(file_path): + self.rm(file_path) + + if os.path.exists(path): + self.rm(path) + + os.makedirs(path) + urlretrieve(download_url, file_path, self.reporthook) + + print("Extracting archive") + if file_name.endswith('.zip'): + opener, mode = zipfile.ZipFile, 'r' + elif file_name.endswith('.tar.gz') or file_name.endswith('.tgz'): + opener, mode = tarfile.open, 'r:gz' + elif file_name.endswith('.tar.bz2') or file_name.endswith('.tbz'): + opener, mode = tarfile.open, 'r:bz2' + else: + raise ValueError("Could not extract `%s` as no appropriate extractor is found" % path) + + cwd = os.getcwd() + os.chdir(path) + + try: + file = opener(file_path, mode) + try: + file.extractall() + finally: + file.close() + finally: + os.chdir(cwd) + + with open(os.path.join(path, ".version"), 'w+') as out: + out.write(self.get_latest_version(organization, repository)) + + return True diff --git a/resource_fetch/setup.py b/resource_fetch/setup.py new file mode 100644 index 00000000..70ceb45c --- /dev/null +++ b/resource_fetch/setup.py @@ -0,0 +1,21 @@ +from setuptools import setup + +with open("README.md", "r") as file: + long_description = file.read() + +setup( + name='resource_fetch', + version='1.0.0', + description='Official CoffeeHouse API Wrapper for Python', + long_description=long_description, + long_description_content_type="text/markdown", + packages=['resource_fetch'], + package_dir={ + 'resource_fetch': 'resource_fetch' + }, + author='Intellivoid Technologies', + author_email='netkas@intellivoid.net', + install_requires=[ + 'githubpy' + ] +)