From 6d8c0495d94abe7b1333824a5bee773822012b60 Mon Sep 17 00:00:00 2001 From: Dan Cojocaru Date: Sun, 22 Aug 2021 15:41:55 +0300 Subject: [PATCH] Added server and Dockerfile --- .dockerignore | 16 + .gitignore | 6 + Dockerfile | 23 + Pipfile | 11 + Pipfile.lock | 20 + {scrapper => scraper}/Pipfile | 0 {scrapper => scraper}/Pipfile.lock | 0 scraper/__init__.py | 1 + {scrapper => scraper}/main.py | 0 {scrapper => scraper}/scraper.py | 25 +- scraper/setup.py | 8 + .../trainInfoScrapResultSchema.json | 31 ++ scrapper/__init__.py | 1 - server/Pipfile | 14 + server/Pipfile.lock | 403 ++++++++++++++++++ server/__init__.py | 1 + server/cache.py | 18 + server/main.py | 18 + server/scraper | 1 + server/server.py | 33 ++ 20 files changed, 617 insertions(+), 13 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 Pipfile create mode 100644 Pipfile.lock rename {scrapper => scraper}/Pipfile (100%) rename {scrapper => scraper}/Pipfile.lock (100%) create mode 100644 scraper/__init__.py rename {scrapper => scraper}/main.py (100%) rename {scrapper => scraper}/scraper.py (87%) create mode 100644 scraper/setup.py rename {scrapper => scraper}/trainInfoScrapResultSchema.json (77%) delete mode 100644 scrapper/__init__.py create mode 100644 server/Pipfile create mode 100644 server/Pipfile.lock create mode 100644 server/__init__.py create mode 100644 server/cache.py create mode 100644 server/main.py create mode 120000 server/scraper create mode 100644 server/server.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..38f9a76 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +# Docker +.dockerignore +Dockerfile + +# CPython compiler output +__pycache__ +*.pyc + +# Python package stuff +reqlib-metadata +pyproject.toml +*.egg-info + +# VS Code +.vscode + diff --git a/.gitignore b/.gitignore index 725e239..19fe82c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ # CPython compiler output +__pycache__ *.pyc +# Python package stuff +reqlib-metadata +pyproject.toml +*.egg-info + # VS Code .vscode diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1c25b40 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:slim + +RUN pip install pipenv + +WORKDIR /var/app/scraper +COPY scraper/Pipfil* ./ +COPY scraper/setup.py ./ +WORKDIR /var/app/server +RUN ln -s /var/app/scraper scraper +COPY server/Pipfil* ./ +RUN pipenv install + +WORKDIR /var/app/scraper +COPY scraper . +WORKDIR /var/app/server +COPY server . +RUN rm scraper +RUN ln -s /var/app/scraper scraper + +ENV PORT 5000 +EXPOSE ${PORT} + +CMD ["pipenv", "run", "python3", "-m", "main"] diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..71e4f7c --- /dev/null +++ b/Pipfile @@ -0,0 +1,11 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..8aee7e8 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,20 @@ +{ + "_meta": { + "hash": { + "sha256": "a36a5392bb1e8bbc06bfaa0761e52593cf2d83b486696bf54667ba8da616c839" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": {} +} diff --git a/scrapper/Pipfile b/scraper/Pipfile similarity index 100% rename from scrapper/Pipfile rename to scraper/Pipfile diff --git a/scrapper/Pipfile.lock b/scraper/Pipfile.lock similarity index 100% rename from scrapper/Pipfile.lock rename to scraper/Pipfile.lock diff --git a/scraper/__init__.py b/scraper/__init__.py new file mode 100644 index 0000000..426b2a3 --- /dev/null +++ b/scraper/__init__.py @@ -0,0 +1 @@ +__all__ = ['scraper'] diff --git a/scrapper/main.py b/scraper/main.py similarity index 100% rename from scrapper/main.py rename to scraper/main.py diff --git a/scrapper/scraper.py b/scraper/scraper.py similarity index 87% rename from scrapper/scraper.py rename to scraper/scraper.py index 5834036..b8c149f 100644 --- a/scrapper/scraper.py +++ b/scraper/scraper.py @@ -7,6 +7,10 @@ import requests from bs4 import BeautifulSoup from urllib.parse import quote, urlencode +TRAIN_INFO_REGEX = re.compile(r'^([A-Z-]+) ([0-9]+) în ([0-9.]+)$') + +OPERATOR_REGEX = re.compile(r'^Operat de (.+)$') + SL_REGEX = re.compile(r'^(?:Fără|([0-9]+) min) (întârziere|mai devreme) la (trecerea fără oprire prin|sosirea în|plecarea din) (.+)\.$') SL_STATE_MAP = { 't': 'passing', @@ -61,17 +65,6 @@ def scrape(train_no: int, use_yesterday=False, date_override=None): soup = BeautifulSoup(r.text, features='html.parser') sform = soup.find(id='form-search') - # required_fields = [ - # 'Date', - # 'TrainRunningNumber', - # 'SelectedBranchCode', - # 'ReCaptcha', - # 'ConfirmationKey', - # 'IsSearchWanted', - # 'IsReCaptchaFailed', - # '__RequestVerificationToken', - # ] - # result_data = { field: sform.find('input', attrs={'name': field})['value'] for field in required_fields } result_data = { elem['name']: elem['value'] for elem in sform('input') } r = s.post('https://mersultrenurilor.infofer.ro/ro-RO/Trains/TrainsResult', data=result_data) @@ -79,7 +72,15 @@ def scrape(train_no: int, use_yesterday=False, date_override=None): scraped = {} - results_div = soup('div', recursive=False)[3].div + train_info_div, _, _, results_div, *_ = soup('div', recursive=False) + + train_info_div = train_info_div.div('div', recursive=False)[0] + + scraped['rank'], scraped['number'], scraped['date'] = TRAIN_INFO_REGEX.match(collapse_space(train_info_div.h2.text)).groups() + + scraped['operator'] = OPERATOR_REGEX.match(collapse_space(train_info_div.p.text)).groups()[0] + + results_div = results_div.div status_div = results_div('div', recursive=False)[0] route_text = collapse_space(status_div.h4.text) route_from, route_to = ROUTE_REGEX.match(route_text).groups() diff --git a/scraper/setup.py b/scraper/setup.py new file mode 100644 index 0000000..e4c6932 --- /dev/null +++ b/scraper/setup.py @@ -0,0 +1,8 @@ +from distutils.core import setup + +setup( + name='InfoFer_Scraper', + version='0.1', + author='Dan Cojocaru', + install_requires=['beautifulsoup4', 'requests'] +) \ No newline at end of file diff --git a/scrapper/trainInfoScrapResultSchema.json b/scraper/trainInfoScrapResultSchema.json similarity index 77% rename from scrapper/trainInfoScrapResultSchema.json rename to scraper/trainInfoScrapResultSchema.json index ec624f2..c94eb74 100644 --- a/scrapper/trainInfoScrapResultSchema.json +++ b/scraper/trainInfoScrapResultSchema.json @@ -34,6 +34,37 @@ }, "type": "object", "properties": { + "rank": { + "description": "The rank of the train", + "type": "string", + "examples": [ + "74", + "15934" + ] + }, + "number": { + "description": "The number of the train", + "type": "string", + "examples": [ + "R", + "R-E", + "IR", + "IRN" + ] + }, + "date": { + "description": "Date of departure from the first station", + "type": "string" + }, + "operator": { + "description": "Operator of the train", + "type": "string", + "examples": [ + "CFR Călători", + "Softrans", + "Regio Călători" + ] + }, "route": { "description": "Route of the train", "type": "object", diff --git a/scrapper/__init__.py b/scrapper/__init__.py deleted file mode 100644 index 448bf8b..0000000 --- a/scrapper/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['scrapper'] diff --git a/server/Pipfile b/server/Pipfile new file mode 100644 index 0000000..48a585d --- /dev/null +++ b/server/Pipfile @@ -0,0 +1,14 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +flask = "*" +gevent = "*" +scraper = { editable = true, path = '../scraper' } + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/server/Pipfile.lock b/server/Pipfile.lock new file mode 100644 index 0000000..e5e4153 --- /dev/null +++ b/server/Pipfile.lock @@ -0,0 +1,403 @@ +{ + "_meta": { + "hash": { + "sha256": "9d422680ab15ce184b043276f5d0d2cac228ff60dfc66ec193b6314bdc0f6ce2" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "beautifulsoup4": { + "hashes": [ + "sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35", + "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25", + "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666" + ], + "version": "==4.9.3" + }, + "certifi": { + "hashes": [ + "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", + "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" + ], + "version": "==2021.5.30" + }, + "cffi": { + "hashes": [ + "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d", + "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771", + "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872", + "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c", + "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc", + "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762", + "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202", + "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5", + "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548", + "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a", + "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f", + "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20", + "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218", + "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c", + "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e", + "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56", + "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224", + "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a", + "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2", + "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a", + "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819", + "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346", + "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b", + "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e", + "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534", + "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb", + "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0", + "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156", + "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd", + "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87", + "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc", + "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195", + "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33", + "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f", + "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d", + "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd", + "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728", + "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7", + "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca", + "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99", + "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf", + "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e", + "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c", + "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5", + "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69" + ], + "markers": "platform_python_implementation == 'CPython' and sys_platform == 'win32'", + "version": "==1.14.6" + }, + "charset-normalizer": { + "hashes": [ + "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b", + "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3" + ], + "markers": "python_version >= '3'", + "version": "==2.0.4" + }, + "click": { + "hashes": [ + "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a", + "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6" + ], + "markers": "python_version >= '3.6'", + "version": "==8.0.1" + }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "platform_system == 'Windows'", + "version": "==0.4.4" + }, + "flask": { + "hashes": [ + "sha256:1c4c257b1892aec1398784c63791cbaa43062f1f7aeb555c4da961b20ee68f55", + "sha256:a6209ca15eb63fc9385f38e452704113d679511d9574d09b2cf9183ae7d20dc9" + ], + "index": "pypi", + "version": "==2.0.1" + }, + "gevent": { + "hashes": [ + "sha256:02d1e8ca227d0ab0b7917fd7e411f9a534475e0a41fb6f434e9264b20155201a", + "sha256:0c7b4763514fec74c9fe6ad10c3de62d8fe7b926d520b1e35eb6887181b954ff", + "sha256:1c9c87b15f792af80edc950a83ab8ef4f3ba3889712211c2c42740ddb57b5492", + "sha256:23077d87d1589ac141c22923fd76853d2cc5b7e3c5e1f1f9cdf6ff23bc9790fc", + "sha256:37a469a99e6000b42dd0b9bbd9d716dbd66cdc6e5738f136f6a266c29b90ee99", + "sha256:3b600145dc0c5b39c6f89c2e91ec6c55eb0dd52dc8148228479ca42cded358e4", + "sha256:3f5ba654bdd3c774079b553fef535ede5b52c7abd224cb235a15da90ae36251b", + "sha256:43e93e1a4738c922a2416baf33f0afb0a20b22d3dba886720bc037cd02a98575", + "sha256:473f918bdf7d2096e391f66bd8ce1e969639aa235e710aaf750a37774bb585bd", + "sha256:4c94d27be9f0439b28eb8bd0f879e6142918c62092fda7fb96b6d06f01886b94", + "sha256:55ede95f41b74e7506fab293ad04cc7fc2b6f662b42281e9f2d668ad3817b574", + "sha256:6cad37a55e904879beef2a7e7c57c57d62fde2331fef1bec7f2b2a7ef14da6a2", + "sha256:72d4c2a8e65bbc702db76456841c7ddd6de2d9ab544a24aa74ad9c2b6411a269", + "sha256:75c29ed5148c916021d39d2fac90ccc0e19adf854626a34eaee012aa6b1fcb67", + "sha256:84e1af2dfb4ea9495cb914b00b6303ca0d54bf0a92e688a17e60f6b033873df2", + "sha256:8d8655ce581368b7e1ab42c8a3a166c0b43ea04e59970efbade9448864585e99", + "sha256:90131877d3ce1a05da1b718631860815b89ff44e93c42d168c9c9e8893b26318", + "sha256:9d46bea8644048ceac5737950c08fc89c37a66c34a56a6c9e3648726e60cb767", + "sha256:a8656d6e02bf47d7fa47728cf7a7cbf408f77ef1fad12afd9e0e3246c5de1707", + "sha256:aaf1451cd0d9c32f65a50e461084a0540be52b8ea05c18669c95b42e1f71592a", + "sha256:afc877ff4f277d0e51a1206d748fdab8c1e0256f7a05e1b1067abbed71c64da9", + "sha256:b10c3326edb76ec3049646dc5131608d6d3733b5adfc75d34852028ecc67c52c", + "sha256:ceec7c5f15fb2f9b767b194daa55246830db6c7c3c2f0b1c7e9e90cb4d01f3f9", + "sha256:e00dc0450f79253b7a3a7f2a28e6ca959c8d0d47c0f9fa2c57894c7974d5965f", + "sha256:e91632fdcf1c9a33e97e35f96edcbdf0b10e36cf53b58caa946dca4836bb688c", + "sha256:f39d5defda9443b5fb99a185050e94782fe7ac38f34f751b491142216ad23bc7" + ], + "index": "pypi", + "version": "==21.8.0" + }, + "greenlet": { + "hashes": [ + "sha256:04e1849c88aa56584d4a0a6e36af5ec7cc37993fdc1fda72b56aa1394a92ded3", + "sha256:05e72db813c28906cdc59bd0da7c325d9b82aa0b0543014059c34c8c4ad20e16", + "sha256:07e6d88242e09b399682b39f8dfa1e7e6eca66b305de1ff74ed9eb1a7d8e539c", + "sha256:090126004c8ab9cd0787e2acf63d79e80ab41a18f57d6448225bbfcba475034f", + "sha256:1796f2c283faab2b71c67e9b9aefb3f201fdfbee5cb55001f5ffce9125f63a45", + "sha256:2f89d74b4f423e756a018832cd7a0a571e0a31b9ca59323b77ce5f15a437629b", + "sha256:34e6675167a238bede724ee60fe0550709e95adaff6a36bcc97006c365290384", + "sha256:3e594015a2349ec6dcceda9aca29da8dc89e85b56825b7d1f138a3f6bb79dd4c", + "sha256:3f8fc59bc5d64fa41f58b0029794f474223693fd00016b29f4e176b3ee2cfd9f", + "sha256:3fc6a447735749d651d8919da49aab03c434a300e9f0af1c886d560405840fd1", + "sha256:40abb7fec4f6294225d2b5464bb6d9552050ded14a7516588d6f010e7e366dcc", + "sha256:44556302c0ab376e37939fd0058e1f0db2e769580d340fb03b01678d1ff25f68", + "sha256:476ba9435afaead4382fbab8f1882f75e3fb2285c35c9285abb3dd30237f9142", + "sha256:4870b018ca685ff573edd56b93f00a122f279640732bb52ce3a62b73ee5c4a92", + "sha256:4adaf53ace289ced90797d92d767d37e7cdc29f13bd3830c3f0a561277a4ae83", + "sha256:4eae94de9924bbb4d24960185363e614b1b62ff797c23dc3c8a7c75bbb8d187e", + "sha256:5317701c7ce167205c0569c10abc4bd01c7f4cf93f642c39f2ce975fa9b78a3c", + "sha256:5c3b735ccf8fc8048664ee415f8af5a3a018cc92010a0d7195395059b4b39b7d", + "sha256:5cde7ee190196cbdc078511f4df0be367af85636b84d8be32230f4871b960687", + "sha256:655ab836324a473d4cd8cf231a2d6f283ed71ed77037679da554e38e606a7117", + "sha256:6ce9d0784c3c79f3e5c5c9c9517bbb6c7e8aa12372a5ea95197b8a99402aa0e6", + "sha256:6e0696525500bc8aa12eae654095d2260db4dc95d5c35af2b486eae1bf914ccd", + "sha256:75ff270fd05125dce3303e9216ccddc541a9e072d4fc764a9276d44dee87242b", + "sha256:8039f5fe8030c43cd1732d9a234fdcbf4916fcc32e21745ca62e75023e4d4649", + "sha256:84488516639c3c5e5c0e52f311fff94ebc45b56788c2a3bfe9cf8e75670f4de3", + "sha256:84782c80a433d87530ae3f4b9ed58d4a57317d9918dfcc6a59115fa2d8731f2c", + "sha256:8ddb38fb6ad96c2ef7468ff73ba5c6876b63b664eebb2c919c224261ae5e8378", + "sha256:98b491976ed656be9445b79bc57ed21decf08a01aaaf5fdabf07c98c108111f6", + "sha256:990e0f5e64bcbc6bdbd03774ecb72496224d13b664aa03afd1f9b171a3269272", + "sha256:9b02e6039eafd75e029d8c58b7b1f3e450ca563ef1fe21c7e3e40b9936c8d03e", + "sha256:a11b6199a0b9dc868990456a2667167d0ba096c5224f6258e452bfbe5a9742c5", + "sha256:a414f8e14aa7bacfe1578f17c11d977e637d25383b6210587c29210af995ef04", + "sha256:a91ee268f059583176c2c8b012a9fce7e49ca6b333a12bbc2dd01fc1a9783885", + "sha256:ac991947ca6533ada4ce7095f0e28fe25d5b2f3266ad5b983ed4201e61596acf", + "sha256:b050dbb96216db273b56f0e5960959c2b4cb679fe1e58a0c3906fa0a60c00662", + "sha256:b97a807437b81f90f85022a9dcfd527deea38368a3979ccb49d93c9198b2c722", + "sha256:bad269e442f1b7ffa3fa8820b3c3aa66f02a9f9455b5ba2db5a6f9eea96f56de", + "sha256:bf3725d79b1ceb19e83fb1aed44095518c0fcff88fba06a76c0891cfd1f36837", + "sha256:c0f22774cd8294078bdf7392ac73cf00bfa1e5e0ed644bd064fdabc5f2a2f481", + "sha256:c1862f9f1031b1dee3ff00f1027fcd098ffc82120f43041fe67804b464bbd8a7", + "sha256:c8d4ed48eed7414ccb2aaaecbc733ed2a84c299714eae3f0f48db085342d5629", + "sha256:cf31e894dabb077a35bbe6963285d4515a387ff657bd25b0530c7168e48f167f", + "sha256:d15cb6f8706678dc47fb4e4f8b339937b04eda48a0af1cca95f180db552e7663", + "sha256:dfcb5a4056e161307d103bc013478892cfd919f1262c2bb8703220adcb986362", + "sha256:e02780da03f84a671bb4205c5968c120f18df081236d7b5462b380fd4f0b497b", + "sha256:e2002a59453858c7f3404690ae80f10c924a39f45f6095f18a985a1234c37334", + "sha256:e22a82d2b416d9227a500c6860cf13e74060cf10e7daf6695cbf4e6a94e0eee4", + "sha256:e41f72f225192d5d4df81dad2974a8943b0f2d664a2a5cfccdf5a01506f5523c", + "sha256:f253dad38605486a4590f9368ecbace95865fea0f2b66615d121ac91fd1a1563", + "sha256:fddfb31aa2ac550b938d952bca8a87f1db0f8dc930ffa14ce05b5c08d27e7fd1" + ], + "markers": "platform_python_implementation == 'CPython'", + "version": "==1.1.1" + }, + "idna": { + "hashes": [ + "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", + "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" + ], + "markers": "python_version >= '3'", + "version": "==3.2" + }, + "infofer-scraper": { + "editable": true, + "path": "./../scraper" + }, + "itsdangerous": { + "hashes": [ + "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c", + "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "jinja2": { + "hashes": [ + "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4", + "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.1" + }, + "markupsafe": { + "hashes": [ + "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", + "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", + "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", + "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", + "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", + "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724", + "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", + "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646", + "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", + "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6", + "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6", + "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad", + "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", + "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38", + "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac", + "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", + "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6", + "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", + "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", + "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", + "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", + "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a", + "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", + "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9", + "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864", + "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", + "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", + "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", + "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", + "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", + "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b", + "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", + "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", + "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", + "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", + "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28", + "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", + "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", + "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d", + "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", + "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", + "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145", + "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", + "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c", + "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1", + "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", + "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53", + "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134", + "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85", + "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", + "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", + "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", + "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", + "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "pycparser": { + "hashes": [ + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.20" + }, + "requests": { + "hashes": [ + "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", + "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==2.26.0" + }, + "scraper": { + "editable": true, + "path": "../scraper" + }, + "soupsieve": { + "hashes": [ + "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc", + "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b" + ], + "markers": "python_version >= '3'", + "version": "==2.2.1" + }, + "urllib3": { + "hashes": [ + "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", + "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.6" + }, + "werkzeug": { + "hashes": [ + "sha256:1de1db30d010ff1af14a009224ec49ab2329ad2cde454c8a708130642d579c42", + "sha256:6c1ec500dcdba0baa27600f6a22f6333d8b662d22027ff9f6202e3367413caa8" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "zope.event": { + "hashes": [ + "sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42", + "sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330" + ], + "version": "==4.5.0" + }, + "zope.interface": { + "hashes": [ + "sha256:08f9636e99a9d5410181ba0729e0408d3d8748026ea938f3b970a0249daa8192", + "sha256:0b465ae0962d49c68aa9733ba92a001b2a0933c317780435f00be7ecb959c702", + "sha256:0cba8477e300d64a11a9789ed40ee8932b59f9ee05f85276dbb4b59acee5dd09", + "sha256:0cee5187b60ed26d56eb2960136288ce91bcf61e2a9405660d271d1f122a69a4", + "sha256:0ea1d73b7c9dcbc5080bb8aaffb776f1c68e807767069b9ccdd06f27a161914a", + "sha256:0f91b5b948686659a8e28b728ff5e74b1be6bf40cb04704453617e5f1e945ef3", + "sha256:15e7d1f7a6ee16572e21e3576d2012b2778cbacf75eb4b7400be37455f5ca8bf", + "sha256:17776ecd3a1fdd2b2cd5373e5ef8b307162f581c693575ec62e7c5399d80794c", + "sha256:194d0bcb1374ac3e1e023961610dc8f2c78a0f5f634d0c737691e215569e640d", + "sha256:1c0e316c9add0db48a5b703833881351444398b04111188069a26a61cfb4df78", + "sha256:205e40ccde0f37496904572035deea747390a8b7dc65146d30b96e2dd1359a83", + "sha256:273f158fabc5ea33cbc936da0ab3d4ba80ede5351babc4f577d768e057651531", + "sha256:2876246527c91e101184f63ccd1d716ec9c46519cc5f3d5375a3351c46467c46", + "sha256:2c98384b254b37ce50eddd55db8d381a5c53b4c10ee66e1e7fe749824f894021", + "sha256:2e5a26f16503be6c826abca904e45f1a44ff275fdb7e9d1b75c10671c26f8b94", + "sha256:334701327f37c47fa628fc8b8d28c7d7730ce7daaf4bda1efb741679c2b087fc", + "sha256:3748fac0d0f6a304e674955ab1365d515993b3a0a865e16a11ec9d86fb307f63", + "sha256:3c02411a3b62668200910090a0dff17c0b25aaa36145082a5a6adf08fa281e54", + "sha256:3dd4952748521205697bc2802e4afac5ed4b02909bb799ba1fe239f77fd4e117", + "sha256:3f24df7124c323fceb53ff6168da70dbfbae1442b4f3da439cd441681f54fe25", + "sha256:469e2407e0fe9880ac690a3666f03eb4c3c444411a5a5fddfdabc5d184a79f05", + "sha256:4de4bc9b6d35c5af65b454d3e9bc98c50eb3960d5a3762c9438df57427134b8e", + "sha256:5208ebd5152e040640518a77827bdfcc73773a15a33d6644015b763b9c9febc1", + "sha256:52de7fc6c21b419078008f697fd4103dbc763288b1406b4562554bd47514c004", + "sha256:5bb3489b4558e49ad2c5118137cfeaf59434f9737fa9c5deefc72d22c23822e2", + "sha256:5dba5f530fec3f0988d83b78cc591b58c0b6eb8431a85edd1569a0539a8a5a0e", + "sha256:5dd9ca406499444f4c8299f803d4a14edf7890ecc595c8b1c7115c2342cadc5f", + "sha256:5f931a1c21dfa7a9c573ec1f50a31135ccce84e32507c54e1ea404894c5eb96f", + "sha256:63b82bb63de7c821428d513607e84c6d97d58afd1fe2eb645030bdc185440120", + "sha256:66c0061c91b3b9cf542131148ef7ecbecb2690d48d1612ec386de9d36766058f", + "sha256:6f0c02cbb9691b7c91d5009108f975f8ffeab5dff8f26d62e21c493060eff2a1", + "sha256:71aace0c42d53abe6fc7f726c5d3b60d90f3c5c055a447950ad6ea9cec2e37d9", + "sha256:7d97a4306898b05404a0dcdc32d9709b7d8832c0c542b861d9a826301719794e", + "sha256:7df1e1c05304f26faa49fa752a8c690126cf98b40b91d54e6e9cc3b7d6ffe8b7", + "sha256:8270252effc60b9642b423189a2fe90eb6b59e87cbee54549db3f5562ff8d1b8", + "sha256:867a5ad16892bf20e6c4ea2aab1971f45645ff3102ad29bd84c86027fa99997b", + "sha256:877473e675fdcc113c138813a5dd440da0769a2d81f4d86614e5d62b69497155", + "sha256:8892f89999ffd992208754851e5a052f6b5db70a1e3f7d54b17c5211e37a98c7", + "sha256:9a9845c4c6bb56e508651f005c4aeb0404e518c6f000d5a1123ab077ab769f5c", + "sha256:a1e6e96217a0f72e2b8629e271e1b280c6fa3fe6e59fa8f6701bec14e3354325", + "sha256:a8156e6a7f5e2a0ff0c5b21d6bcb45145efece1909efcbbbf48c56f8da68221d", + "sha256:a9506a7e80bcf6eacfff7f804c0ad5350c8c95b9010e4356a4b36f5322f09abb", + "sha256:af310ec8335016b5e52cae60cda4a4f2a60a788cbb949a4fbea13d441aa5a09e", + "sha256:b0297b1e05fd128d26cc2460c810d42e205d16d76799526dfa8c8ccd50e74959", + "sha256:bf68f4b2b6683e52bec69273562df15af352e5ed25d1b6641e7efddc5951d1a7", + "sha256:d0c1bc2fa9a7285719e5678584f6b92572a5b639d0e471bb8d4b650a1a910920", + "sha256:d4d9d6c1a455d4babd320203b918ccc7fcbefe308615c521062bc2ba1aa4d26e", + "sha256:db1fa631737dab9fa0b37f3979d8d2631e348c3b4e8325d6873c2541d0ae5a48", + "sha256:dd93ea5c0c7f3e25335ab7d22a507b1dc43976e1345508f845efc573d3d779d8", + "sha256:f44e517131a98f7a76696a7b21b164bcb85291cee106a23beccce454e1f433a4", + "sha256:f7ee479e96f7ee350db1cf24afa5685a5899e2b34992fb99e1f7c1b0b758d263" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==5.4.0" + } + }, + "develop": {} +} diff --git a/server/__init__.py b/server/__init__.py new file mode 100644 index 0000000..7cd07a9 --- /dev/null +++ b/server/__init__.py @@ -0,0 +1 @@ +__all__ = ['server'] diff --git a/server/cache.py b/server/cache.py new file mode 100644 index 0000000..d7e887d --- /dev/null +++ b/server/cache.py @@ -0,0 +1,18 @@ +from datetime import date, datetime, timedelta + +_NO_DEFAULT = object() + +class CachedData: + def __init__(self, getter, initial_data=_NO_DEFAULT, validity=1000): + self.getter = getter + self.data = initial_data + self.last_refresh_date = datetime.now() + self.validity = timedelta(milliseconds=validity) + if initial_data == _NO_DEFAULT: + self.last_refresh_date -= self.validity + + def __call__(self, *args, **kwds): + if self.last_refresh_date + self.validity < datetime.now(): + self.data = self.getter() + self.last_refresh_date = datetime.now() + return self.data, self.last_refresh_date diff --git a/server/main.py b/server/main.py new file mode 100644 index 0000000..d427140 --- /dev/null +++ b/server/main.py @@ -0,0 +1,18 @@ +from gevent.pywsgi import WSGIServer +from server import app + +def main(): + port = 5000 + + import os + try: + port = int(os.environ['PORT']) + except: + pass + + print(f'Starting server on port {port}') + http_server = WSGIServer(('', port), app) + http_server.serve_forever() + +if __name__ == '__main__': + main() diff --git a/server/scraper b/server/scraper new file mode 120000 index 0000000..6954597 --- /dev/null +++ b/server/scraper @@ -0,0 +1 @@ +C:/Users/kbruen/LaptopSync/CFR/new_infofer_scraper/scraper \ No newline at end of file diff --git a/server/server.py b/server/server.py new file mode 100644 index 0000000..8fccdd3 --- /dev/null +++ b/server/server.py @@ -0,0 +1,33 @@ +from flask import Flask, json, request, jsonify + +from cache import CachedData + +app = Flask(__name__) + +@app.route('/') +def root(): + return 'Test' + +train_data_cache = {} + +@app.route('/train/') +def get_train_info(train_no: int): + def get_data(): + print(f'Cache miss for {train_no}') + from scraper.scraper import scrape + use_yesterday = False + return scrape(train_no, use_yesterday=use_yesterday) + if train_no not in train_data_cache: + train_data_cache[train_no] = CachedData(get_data, validity=1000 * 30) + data, fetch_time = train_data_cache[train_no]() + resp = jsonify(data) + resp.headers['X-Last-Fetched'] = fetch_time.isoformat() + return resp + +@app.route('/trains') +def get_trains(): + return jsonify(list(train_data_cache.keys())) + +if __name__ == '__main__': + print('Starting debug server on port 5001') + app.run(port=5000)