TVSCREENER-27 Generate field values

houseofai · Aug 11, 2023 · efc2c8b · efc2c8b
1 parent 14b8279
commit efc2c8b
Show file tree

Hide file tree

Showing 41 changed files with 2,461 additions and 589 deletions.
diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
@@ -2,32 +2,32 @@ name: Codecov
 
 on:
   pull_request:
-      branches:
+    branches:
       - master
 
 jobs:
   run:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
-    - name: Setup Python
-      uses: actions/setup-python@master
-      with:
-        python-version: '3.x'
-    - name: Generate coverage report
-      run: |
-        pip install -r requirements.txt
-        pip install pytest
-        pip install pytest-cov
-        pytest --cov=./ --cov-report=xml
-    - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        #directory: ./coverage/reports/
-        env_vars: OS,PYTHON
-        fail_ci_if_error: true
-        files: ./coverage.xml
-        flags: unittests
-        name: codecov-umbrella
-        verbose: true
+      - uses: actions/checkout@master
+      - name: Setup Python
+        uses: actions/setup-python@master
+        with:
+          python-version: '3.x'
+      - name: Generate coverage report
+        run: |
+          pip install -r requirements.txt
+          pip install pytest
+          pip install pytest-cov
+          pytest --cov=./ --cov-report=xml
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          #directory: ./coverage/reports/
+          env_vars: OS,PYTHON
+          fail_ci_if_error: true
+          files: ./coverage.xml
+          flags: unittests
+          name: codecov-umbrella
+          verbose: true
diff --git a/.github/workflows/pip-publish.yml b/.github/workflows/pip-publish.yml
@@ -10,7 +10,7 @@ name: Upload Python Package
 
 on:
   release:
-    types: [published]
+    types: [ published ]
 
 permissions:
   contents: read
@@ -21,19 +21,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v3
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build
-    - name: Build package
-      run: python -m build
-    - name: Publish package
-      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+      - name: Build package
+        run: python -m build
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/README.md b/README.md
@@ -16,31 +16,36 @@ Get the results as a Pandas Dataframe
 ![dataframe.png](https://github.com/houseofai/tvscreener/blob/main/.github/img/dataframe.png?raw=true)
 
 # Main Features
+
 - Query **Stock**, **Forex** and **Crypto** Screener
 - All the **fields available**: ~300 fields - even hidden ones)
 - **Any time interval** (`no need to be a registered user` - 1D, 5m, 1h, etc.)
 - Filters by any fields, symbols, markets, countries, etc.
 - Get the results as a Pandas Dataframe
 
-
 ## Installation
+
 The source code is currently hosted on GitHub at:
 https://github.com/houseofai/tvscreener
 
 Binary installers for the latest released version are available at the [Python
 Package Index (PyPI)](https://pypi.org/project/tvscreener)
+
 ```sh
 # or PyPI
 pip install tvscreener
 ```
+
 From pip + GitHub:
+
 ```sh
 $ pip install git+https://github.com/houseofai/tradingview-screener@main
 ```
 
 ## Usage
 
 For Stocks screener:
+
 ```python
 import tvscreener as tvs
 
@@ -49,14 +54,18 @@ df = ss.get()
 
 # ... returns a dataframe with 150 rows by default
 ``` 
+
 For Forex screener:
+
 ```python
 import tvscreener as tvs
 
 fs = tvs.ForexScreener()
 df = fs.get()
 ```
+
 For Crypto screener:
+
 ```python
 import tvscreener as tvs
 
@@ -66,4 +75,5 @@ df = cs.get()
 
 ## Parameters
 
-For Options and Filters, please check the [notebooks](https://github.com/houseofai/tvscreener/tree/main/notebooks) for examples.
+For Options and Filters, please check the [notebooks](https://github.com/houseofai/tvscreener/tree/main/notebooks) for
+examples.
diff --git a/generate/Generate FilterFields.ipynb b/generate/Generate FilterFields.ipynb
@@ -0,0 +1,175 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2023-08-11T06:52:01.213849587Z",
+     "start_time": "2023-08-11T06:52:01.172112997Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
+   "source": [
+    "from generate import scrap_field_values\n",
+    "import json\n",
+    "\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "outputs": [],
+   "source": [
+    "from generate import fill_filter_template, generate_filter_columns\n",
+    "from generate import write\n",
+    "\n",
+    "\n",
+    "def save_columns(url):\n",
+    "    fields_dict = scrap_field_values(url)\n",
+    "    for key, value in fields_dict.items():\n",
+    "        with open(f'data/{key.lower()}.json', 'w') as f:\n",
+    "            json.dump({key: value}, f)\n",
+    "    return fields_dict\n",
+    "        \n",
+    "def generate_code_files(name):\n",
+    "    with open(f'data/{name.lower()}.json') as f:\n",
+    "        filters_ = json.load(f)\n",
+    "    formatted_columns = generate_filter_columns(filters_[name])\n",
+    "    template = fill_filter_template(name.replace(' ', ''), formatted_columns)\n",
+    "    write(name.replace(' ', '_').lower(), template)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2023-08-11T06:52:03.190052866Z",
+     "start_time": "2023-08-11T06:52:03.186114291Z"
+    }
+   },
+   "id": "d4c4d0ddb0c0119b"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[1]/div[1] ...\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[2]/div[1] ...\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[3]/div[1] ...\n",
+      "TimeoutException on field:  Primary Listing\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[4]/div[1] ...\n",
+      "TimeoutException on field:  Current trading day\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[5]/div[1] ...\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[6]/div[1] ...\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[7]/div[1] ...\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[8]/div[1] ...\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[9]/div[1] ...\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[10]/div[1] ...\n",
+      "NoSuchElementException on field:  Market Capitalization\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[11]/div[1] ...\n",
+      "NoSuchElementException on field:  Volume\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[12]/div[1] ...\n",
+      "NoSuchElementException on field:  Average Volume (10 day)\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[13]/div[1] ...\n",
+      "NoSuchElementException on field:  Average Volume (30 day)\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[14]/div[1] ...\n",
+      "NoSuchElementException on field:  Average Volume (60 day)\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[15]/div[1] ...\n",
+      "NoSuchElementException on field:  Average Volume (90 day)\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[16]/div[1] ...\n",
+      "TimeoutException on field:  Relative Volume\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[17]/div[1] ...\n",
+      "TimeoutException on field:  Relative Volume at Time\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[18]/div[1] ...\n",
+      "TimeoutException on field:  Change %\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[19]/div[1] ...\n",
+      "TimeoutException on field:  Change 1m, %\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[20]/div[1] ...\n",
+      "TimeoutException on field:  Change 5m, %\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[21]/div[1] ...\n",
+      "TimeoutException on field:  Change 15m, %\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[22]/div[1] ...\n",
+      "TimeoutException on field:  Change 1h, %\n",
+      "Search field on xpath:  /html/body/div[10]/div/div/div/div[3]/div[1]/div/div/div[23]/div[1] ...\n"
+     ]
+    }
+   ],
+   "source": [
+    "stock_url = \"https://www.tradingview.com/screener/\"\n",
+    "filters = save_columns(stock_url)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2023-08-11T06:52:42.294915919Z",
+     "start_time": "2023-08-11T06:52:04.108834356Z"
+    }
+   },
+   "id": "e4eabbd29fc14556"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "outputs": [],
+   "source": [
+    "for key in filters.keys():\n",
+    "    generate_code_files(key)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2023-08-11T07:05:13.158804479Z",
+     "start_time": "2023-08-11T07:05:13.148339900Z"
+    }
+   },
+   "id": "fce7ec0c7477cb3c"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false
+   },
+   "id": "c8b265fdafe0ea0"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}