Added ResourceFetch

Makefile

@@ -16,6 +16,9 @@ clean_nlpfr:
 clean_alg:
 	rm -rf alg/build alg/dist alg/coffeehouse_alg.egg-info
 
+clean_rf:
+	rm -rf resource_fetch/build resource_fetch/dist resource_fetch/resource_fetch.egg-info
+
 clean_his:
 	rm -rf hyper_internal_service/build hyper_internal_service/dist hyper_internal_service/hyper_internal_service.egg-info
 
@@ -24,68 +27,82 @@ clean:
 	make clean_dltc
 	make clean_his
 	make clean_alg
+	make clean_rf
 
 # ======================================================================================================================
 
 build_apt:
-	cd mods/apt; python3.6 setup.py build; python3.6 setup.py sdist
+	cd mods/apt; python3 setup.py build; python3 setup.py sdist
 
 build_stopwords:
-	cd mods/stopwords; python3.6 setup.py build; python3.6 setup.py sdist
+	cd mods/stopwords; python3 setup.py build; python3 setup.py sdist
 
 build_tokenizer:
-	cd mods/tokenizer; python3.6 setup.py build; python3.6 setup.py sdist
+	cd mods/tokenizer; python3 setup.py build; python3 setup.py sdist
 
 build_mods:
 	make build_apt build_stopwords build_tokenizer
 
 build_nlpfr:
 	make build_mods
-	cd nlpfr; python3.6 setup.py build; python3.6 setup.py sdist
+	cd nlpfr; python3 setup.py build; python3 setup.py sdist
 
 build_dltc:
-	cd dltc; python3.6 setup.py build; python3.6 setup.py sdist
+	cd dltc; python3 setup.py build; python3 setup.py sdist
 
 build_alg:
-	cd alg; python3.6 setup.py build; python3.6 setup.py sdist
+	cd alg; python3 setup.py build; python3 setup.py sdist
 
 build_his:
-	cd hyper_internal_service; python3.6 setup.py build; python3.6 setup.py sdist
+	cd hyper_internal_service; python3 setup.py build; python3 setup.py sdist
+
+build_rf:
+	cd resource_fetch; python3 setup.py build; python3 setup.py sdist
 
 build:
 	make build_nlpfr
 	make build_his
 	make build_dltc
 	make build_alg
+	make build_rf
 
 # ======================================================================================================================
 
+install_requirements:
+	python3 -m pip install -Ur requirements.txt
+
 install_apt:
-	cd mods/apt; python3.6 setup.py install
+	cd mods/apt; python3 setup.py install
 
 install_stopwords:
-	cd mods/stopwords; python3.6 setup.py install
+	cd mods/stopwords; python3 setup.py install
 
 install_tokenizer:
-	cd mods/tokenizer; python3.6 setup.py install
+	cd mods/tokenizer; python3 setup.py install
 
 install_mods:
 	make install_apt install_stopwords install_tokenizer
 
 install_nlpfr:
 	make install_mods
-	cd nlpfr; python3.6 setup.py install
+	cd nlpfr; python3 setup.py install
 
 install_dltc:
-	cd dltc; python3.6 setup.py install
+	cd dltc; python3 setup.py install
 
 install_alg:
-	cd alg; python3.6 setup.py install
+	cd alg; python3 setup.py install
 
 install_his:
-	cd hyper_internal_service; python3.6 -m pip install -Ur dev_requirements.txt; python3.6 setup.py install
+	cd hyper_internal_service; python3 -m pip install -Ur dev_requirements.txt; python3 setup.py install
+
+install_rf:
+	# Make sure to run 'python3 -m resource_fetch "<PERSONAL ACCESS TOKEN>"' before using resource_fetch!
+	cd resource_fetch; python3 setup.py install
 
 install:
+	make install_requirements
+	make install_rf
 	make install_nlpfr
 	make install_his
 	make install_dltc
@@ -94,13 +111,13 @@ install:
 # ======================================================================================================================
 
 system_prep_python:
-	apt -y install python3.6 python3-distutils python3-dev python3-setuptools
+	apt -y install python3 python3-distutils python3-dev python3-setuptools
 
 system_prep_pip:
 	apt -y install wget curl
 	wget https://bootstrap.pypa.io/get-pip.py
-	python3.6 get-pip.py
+	python3 get-pip.py
 	rm get-pip.py
 
 system_prep_gcc:
-	apt -y install gcc build-essential
+	apt -y install gcc build-essential

requirements.txt

@@ -0,0 +1,24 @@
+setuptools~=50.3.2
+requests~=2.22.0
+Keras~=2.2.5
+numpy~=1.19.4
+nltk~=3.5
+six~=1.15.0
+gensim~=3.8.3
+scikit-learn~=0.22.0
+nose~=1.3.7
+scipy~=1.3.3
+Click~=7.0
+tqdm~=4.51.0
+regex~=2020.11.11
+pyparsing~=2.4.6
+Unidecode~=1.1.1
+packaging~=20.3
+PyGithub~=1.54.1
+multidict~=4.7.6
+yarl~=1.4.2
+attrs~=19.3.0
+gunicorn~=20.0.4
+aiodns~=2.0.0
+cchardet~=2.1.6
+pytest~=5.4.2

resource_fetch/README.md

@@ -0,0 +1,6 @@
+# Resource Fetch
+
+This CoffeeHouse library will allow you to set a Github Personal Access Token and use it
+to fetch releases and extract them onto the system so that it can later be fetched within
+runtime. This library will also keep track of the current version of the resource and update
+it when there's a new version available.

resource_fetch/resource_fetch/__init__.py

@@ -0,0 +1,2 @@
+from . import resource_fetch
+from resource_fetch.resource_fetch import ResourceFetch

resource_fetch/resource_fetch/__main__.py

@@ -0,0 +1,16 @@
+import sys
+
+from resource_fetch import ResourceFetch
+
+
+def main():
+    resource_fetcher = ResourceFetch()
+    if len(sys.argv[1:]) > 0:
+        resource_fetcher.define_pat(sys.argv[1])
+    else:
+        resource_fetcher.define_pat(input("Personal Access Token: "))
+    print("The Personal Access Token has been successfully defined")
+
+
+if __name__ == '__main__':
+    main()

resource_fetch/resource_fetch/resource_fetch.py

@@ -0,0 +1,186 @@
+import os
+import sys
+import tarfile
+import zipfile
+from pathlib import Path
+from packaging import version
+from github import Github
+from urllib.request import urlretrieve
+from urllib.parse import urlparse
+
+
+class ResourceFetch(object):
+
+    def __init__(self):
+        """
+        ResourceFetch constructor
+        """
+
+        self.working_directory = os.path.join("/", "var", "resources")
+        self.pat_path = os.path.join(self.working_directory, ".pat")
+        self.github_client = None
+        if not os.path.exists(self.working_directory):
+            os.makedirs(self.working_directory)
+
+    def rm(self, path):
+        """
+        Recursively removes a directory or file
+
+        :param path:
+        :return:
+        """
+
+        if os.path.isfile(path):
+            os.remove(path)
+            return
+
+        directory = Path(path)
+        for item in directory.iterdir():
+            if item.is_dir():
+                self.rm(item)
+            else:
+                item.unlink()
+        directory.rmdir()
+
+    def get_github(self):
+        if self.github_client is None:
+            self.github_client = Github(self.get_pat())
+        return self.github_client
+
+    def define_pat(self, token):
+        """
+        Defines/overwrites the personal access token for fetching resources
+
+        :param token:
+        :return:
+        """
+
+        if os.path.exists(self.pat_path):
+            self.rm(self.pat_path)
+        with open(self.pat_path, 'w+') as out:
+            out.write(token)
+
+    def get_pat(self):
+        """
+        Returns the personal access token that has been defined for fetching resources
+
+        :return:
+        """
+        if not os.path.exists(self.pat_path):
+            raise PermissionError("No Personal Access Token has been defined for ResourceFetcher")
+        with open(self.pat_path, 'r') as file:
+            return file.read()
+
+    def fetch(self, organization, repository, check_for_update=True):
+        """
+        Fetches a resource, downloads it if it's missing and or updates it if it's outdated
+
+        :param organization:
+        :param repository:
+        :param check_for_update:
+        :return:
+        """
+        resource_path = os.path.join(self.working_directory, repository)
+
+        if os.path.exists(resource_path):
+            # The resource already exists, verify it
+
+            if not check_for_update:  # Skip the check for updates
+                return resource_path
+
+            version_file = os.path.join(resource_path, ".version")
+            with open(version_file, 'r') as file:
+                current_version = file.read()
+
+            # Check if it's outdated
+            latest_version = self.get_latest_version(organization, repository)
+            if version.parse(current_version) < version.parse(latest_version):
+                self.install_resource(organization, repository, resource_path)
+            return resource_path
+        self.install_resource(organization, repository, resource_path)
+        return resource_path
+
+    def get_latest_version(self, organization, repository):
+        """
+        Returns the latest version of the resource
+
+        :param organization:
+        :param repository:
+        :return:
+        """
+        releases = self.get_github().get_user(organization).get_repo(repository).get_releases()
+        return releases[0].title
+
+    def get_latest_download_url(self, organization, repository):
+        """
+        Gets the latest download URL for the requested resource
+
+        :param organization:
+        :param repository:
+        :return:
+        """
+
+        releases = self.get_github().get_user(organization).get_repo(repository).get_releases()
+        return releases[0].get_assets()[0].browser_download_url
+
+    @staticmethod
+    def reporthook(blocknum, blocksize, totalsize):
+        readsofar = blocknum * blocksize
+        if totalsize > 0:
+            percent = readsofar * 1e2 / totalsize
+            s = "\r%5.1f%% %*d / %d" % (
+                percent, len(str(totalsize)), readsofar, totalsize)
+            sys.stderr.write(s)
+            if readsofar >= totalsize:  # near the end
+                sys.stderr.write("\n")
+        else:  # total size is unknown
+            sys.stderr.write("read %d\n" % (readsofar,))
+
+    def install_resource(self, organization, repository, path):
+        print("Installing {0} from {1}".format(repository, organization))
+        temporary_directory = os.path.join(self.working_directory, "tmp")
+        if not os.path.exists(temporary_directory):
+            os.makedirs(temporary_directory)
+
+        # Prepare the download
+        download_url = self.get_latest_download_url(organization, repository)
+        parsed_url = urlparse(download_url)
+        file_name = os.path.basename(parsed_url.path)
+        file_path = os.path.join(temporary_directory, file_name)
+        file_extension = os.path.splitext(file_name)[1]
+
+        if os.path.exists(file_path):
+            self.rm(file_path)
+
+        if os.path.exists(path):
+            self.rm(path)
+
+        os.makedirs(path)
+        urlretrieve(download_url, file_path, self.reporthook)
+
+        print("Extracting archive")
+        if file_name.endswith('.zip'):
+            opener, mode = zipfile.ZipFile, 'r'
+        elif file_name.endswith('.tar.gz') or file_name.endswith('.tgz'):
+            opener, mode = tarfile.open, 'r:gz'
+        elif file_name.endswith('.tar.bz2') or file_name.endswith('.tbz'):
+            opener, mode = tarfile.open, 'r:bz2'
+        else:
+            raise ValueError("Could not extract `%s` as no appropriate extractor is found" % path)
+
+        cwd = os.getcwd()
+        os.chdir(path)
+
+        try:
+            file = opener(file_path, mode)
+            try:
+                file.extractall()
+            finally:
+                file.close()
+        finally:
+            os.chdir(cwd)
+
+        with open(os.path.join(path, ".version"), 'w+') as out:
+            out.write(self.get_latest_version(organization, repository))
+
+        return True

resource_fetch/setup.py

@@ -0,0 +1,21 @@
+from setuptools import setup
+
+with open("README.md", "r") as file:
+    long_description = file.read()
+
+setup(
+    name='resource_fetch',
+    version='1.0.0',
+    description='Official CoffeeHouse API Wrapper for Python',
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    packages=['resource_fetch'],
+    package_dir={
+        'resource_fetch': 'resource_fetch'
+    },
+    author='Intellivoid Technologies',
+    author_email='netkas@intellivoid.net',
+    install_requires=[
+        'githubpy'
+    ]
+)