From bafe3494d58fd6a240ebce598d75a8f4c36a7a8b Mon Sep 17 00:00:00 2001 From: awmatheson Date: Thu, 8 Aug 2024 13:00:27 -0700 Subject: [PATCH] update CI --- .github/workflows/CI.yml | 8 +- pyproject.toml | 2 +- src/bytewax_clickhouse.egg-info/PKG-INFO | 315 ++++++++++++++++++++++- 3 files changed, 316 insertions(+), 9 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ebdf751..16b5d03 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,6 +4,9 @@ on: push: branches: - main + release: + types: + - published pull_request: {} merge_group: workflow_dispatch: @@ -124,14 +127,14 @@ jobs: publish-to-pypi: name: Publish to PyPI - if: startsWith(github.ref, 'refs/tags/') + if: github.event_name == 'release' && github.event.action == 'published' needs: - build - test runs-on: ubuntu-latest environment: name: pypi - url: https://pypi.org/p/bytewax-interval + url: https://pypi.org/p/bytewax-clickhouse permissions: id-token: write # IMPORTANT: mandatory for trusted publishing steps: @@ -144,3 +147,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ # Remove when publishing to PyPI + skip-existing: true diff --git a/pyproject.toml b/pyproject.toml index d434026..a3978ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.0.1" description = "Clickhouse sink connector for bytewax" readme = "README.md" requires-python = ">=3.9" -license = {file = "LICENSE.txt"} +license = {file = "LICENSE.md"} keywords = ["bytewax", "streaming", "operator"] authors = [ {name = "Bytewax, Inc.", email = "info@bytewax.io" } diff --git a/src/bytewax_clickhouse.egg-info/PKG-INFO b/src/bytewax_clickhouse.egg-info/PKG-INFO index a307eff..bb11cfc 100644 --- a/src/bytewax_clickhouse.egg-info/PKG-INFO +++ b/src/bytewax_clickhouse.egg-info/PKG-INFO @@ -3,6 +3,288 @@ Name: bytewax-clickhouse Version: 0.0.1 Summary: Clickhouse sink connector for bytewax Author-email: "Bytewax, Inc." +License: # Software License Agreement + + The terms and conditions of this Software License Agreement (this + "License Agreement") govern your use of the accompanying Bytewax Module + software (the "Software") and constitute a binding legal contract + between you and Bytewax, Inc. ("Bytewax"). Your download, installation + or use of the Software constitutes acceptance of this License Agreement. + If you do not agree to any part of this License Agreement, then you may + not use the Software and must delete all copies of the Software in your + possession or control. + + If you are accepting the terms of this Agreement on behalf of a legal + entity, you represent and warrant that you have the authority to bind + that legal entity to the terms of this License Agreement, and, in such + event, "you" and "your" will refer to that legal entity. If you do not + have such authority, you must not accept this License Agreement, in + which case you may not download or use the Software. + + 1. Subject to all terms and conditions of this License Agreement and + your timely payment of all subscription fees due hereunder, Bytewax + hereby grants you a limited, nonexclusive, nontransferable license + (without right to sublicense) to: (i) install and use the object + code version of the Software internally for the purpose of ingesting + and processing data for your own internal business purposes; + and (ii) incorporate the Software, solely in object code form or + code artifact, as a component of software-as-a-service offering that + is hosted by you ("Application"), provided that such Application + provides material additional functionality and features in addition + to that provided by the Software. + + 2. Subject to all terms and conditions of this License Agreement and + your timely payment of all subscription fees due hereunder, Bytewax + hereby grants you, a limited, nonexclusive, nontransferable license + to internally use any documentation that may be provided by Bytewax + regarding the Software ("Documentation") as reasonably necessary for + your authorized use of the Software. You may make copies of the + Documentation in connection with exercising the foregoing license. + Without limiting any other terms and conditions of this License + Agreement, you may not transfer or provide the Documentation to any + third party. + + 3. You shall be entitled to download and use any updated versions of + the Software that Bytewax may make available to similarly situated + licensees in its sole discretion. Any such updated versions + constitute the "Software" for all purposes hereunder. In addition, + Bytewax will use commercially reasonable efforts to answer questions + regarding use of the Software during Bytewax's normal business hours + through such support channels as Bytewax may make available from + time to time. + + 4. You have no rights or licenses with respect to the Software or + Documentation except as expressly provided in this License + Agreement. Without limiting the generality of the foregoing, you may + not, except to the extent expressly provided for in Sections 1 and 2 + above: (a) copy, distribute, rent, lease, lend, sublicense or + transfer the Software or Documentation or use the Software or + Documentation on a service bureau basis; (b) decompile, reverse + engineer, or disassemble the Software or otherwise attempt to + discover the source code of the Software, except solely to the + extent such acts are authorized under applicable law notwithstanding + this prohibition; (c) create derivative works based on the Software + or Documentation; (d) modify, remove, or obscure any copyright, + trademark, patent or other notices or legends that may appear on the + Software or Documentation or during the use and operation + thereof; (e) cause the Software to become subject to the terms of + any open source license agreement; or (f) deploy your Application in + a manner that allows the end user to directly access, download or + use the Software other than as incorporated within your Application. + You are not required to provide any ideas, feedback or suggestions + regarding the Software (collectively, "Feedback") to Bytewax. To the + extent you do provide any Feedback to Bytewax, you acknowledge that + Bytewax may freely use, reproduce, modify, distribute, make, have + made, sell, offer for sale, import and otherwise exploit in any + manner such Feedback without payment of any royalties or other + consideration to you. + + 5. As part of its operation, the Software may collect and send to + Bytewax data regarding the system on which the Software is installed + and relating to usage of the Software ("Collected Data"). Collected + Data will not include the data that you ingest or process through + use of the Software. Collected Data may be used by or on behalf of + Bytewax for any lawful purpose, including without limitation to + develop and improve Bytewax's products and services and for + statistical analysis purposes. In addition, Bytewax will disclose + Collected Data where Bytewax, in good faith, believes that the law + or legal process (such as a court order, search warrant or subpoena) + requires Bytewax to do so or in other circumstances where Bytewax + believes it is necessary to protect the rights or property of + Bytewax or third parties. + + 6. For the avoidance of doubt, you may not directly or indirectly + utilize the Software as an embedded component of any device or + system or as part of any software application that is distributed. + The Software is licensed solely for your internal use or use in an + Application as set forth in Section 1. If you wish to request a + license for any other purpose, please contact Bytewax at + sales@bytewax.io. + + 7. You may only make your Application available under terms and + conditions that: (i) do not conflict with any provision of this + License Agreement or Bytewax's rights in and to the Software; (ii) + are not materially less protective of the Software and Bytewax's + rights therein and thereto than the provisions of this License + Agreement; (iii) do not make any representations, warranties or + covenants regarding the Software or on behalf of Bytewax; (iv) state + that the Application incorporates the Software and that Bytewax owns + all copyright and other intellectual property rights in and to the + Software; (v) disclaim any and all warranties and liability on + behalf of Bytewax to the maximum extent allowable under applicable + law; and (vi) are presented to and accepted by the Application end + user in a manner sufficient to create a binding legal agreement + under applicable law. + + 8. The Software and Documentation are licensed, not sold. The Software + and Documentation provided hereunder are the property of Bytewax or + its third-party licensors. Bytewax shall retain all right, title and + ownership interest and all worldwide patent, copyright, trade + secret, trademark and other intellectual property rights and other + proprietary rights in and to the Software and Documentation and any + copies, derivative works, upgrades, updates, improvements and + modifications thereof, in, regardless of the form or media in which, + or on which, the original and any other copies, derivative works, + upgrades, updates, improvements or modifications may exist. + + 9. You agree to pay all subscription fees for the Software for the + applicable subscription term you select ("Subscription Term"), as + set forth on our website. Each Subscription Term shall automatically + renew for a Subscription Term of equal length until you cancel the + subscription on or before the end of the then current Subscription + Term, unless otherwise terminated in accordance with this License + Agreement. You agree to provide a valid credit card or other + approved payment mechanism for paying subscription fees and other + charges. You authorize us to charge your credit card or other + payment mechanism for any subscription fees and charges that you may + incur in connection with your account or use of the Software. We + reserve the right to change the fee structure and fees charged for + the Software at any time. We will provide you with thirty days + advance notice (which may be by email) of changes to the fee + structure and fees. By continuing use of the Software following that + thirty day notice period, you agree to the updated fee structure and + fees. You acknowledge that if you object to the proposed change(s), + your sole remedy is to cancel your subscription. All fees are + exclusive of any taxes required by applicable law, and you agree to + pay or reimburse Bytewax for any all taxes relating to this License + Agreement, other than taxes based on Bytewax's net income. + + 10. If you breach any provision of this License Agreement or cancel your + subscription, this License Agreement (including all of your rights + and licenses with respect to the Software and Documentation) shall + immediately terminate without further notice from or action by + Bytewax. Bytewax may decline to renew your subscription and + terminate this License Agreement at the end of the Subscription Term + for any or no reason upon notice (including by email) to you. Upon + termination or expiration of this License Agreement, you agree to + cease all use of the Software and Documentation (including by + removing it from your Application) and to delete all copies thereof + in your possession or control. The provisions of Sections 4, 5, 6, + 8, 9 (with respect to accrued but unpaid fees), 11 through 22 and + this Section 10 shall survive any termination or expiration of this + Agreement according to their terms. + + 11. You agree that Bytewax may state in the public domain that you are a + user of the Software, and may include references to and screenshots + of your Applications in Bytewax's marketing and publicity materials. + + 12. THE SOFTWARE AND DOCUMENTATION ARE FURNISHED "AS IS". BYTEWAX MAKES + NO WARRANTIES, WHETHER EXPRESS OR IMPLIED WITH RESPECT TO THE + SOFTWARE AND THE DOCUMENTATION, INCLUDING WITH RESPECT TO THEIR + QUALITY, PERFORMANCE, MERCHANTABILITY, NON-INFRINGEMENT, OR FITNESS + FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND + PERFORMANCE OF THE SOFTWARE AND DOCUMENTATION IS WITH YOU. BYTEWAX + HAS NO OBLIGATION TO PROVIDE ANY SUPPORT OR ASSISTANCE RELATING TO + YOUR USE OF THE SOFTWARE. + + 13. IN NO EVENT SHALL BYTEWAX BE LIABLE FOR ANY CONSEQUENTIAL, INDIRECT, + SPECIAL, OR INCIDENTAL DAMAGES ARISING OUT OF OR RELATING TO THIS + LICENSE AGREEMENT OR YOUR USE OF THE SOFTWARE, EVEN IF BYTEWAX HAS + BEEN ADVISED OF THE POSSIBILITY OF SUCH POTENTIAL LOSS OR DAMAGE AND + REGARDLESS OF THE THEORY OF LIABILITY. IN NO EVENT SHALL BYTEWAX 'S + AGGREGATE LIABILITY TO YOU IN RELATION TO THIS LICENSE AGREEMENT + EXCEED FEES PAID BY YOU HEREUNDER DURING THE SIX MONTH PERIOD + PRECEDING THE DATE OF THE CLAIM. THE FOREGOING LIMITATIONS OF + LIABILITY AND EXCLUSIONS OF CERTAIN DAMAGES SHALL APPLY REGARDLESS + OF THE EXISTENCE OF MULTIPLE CLAIMS OR THE THEORY OF LIABILITY, AND + REGARDLESS OF THE SUCCESS OR EFFECTIVENESS OF OTHER REMEDIES. + + 14. You agree to indemnify and hold Bytewax and its affiliated + companies, and each of their directors, officers, employees, + contractors, suppliers and partners, harmless from any claims, + losses, damages, liabilities, costs and expenses, including + attorney's fees, arising out of or relating to your use or misuse of + the Software (including without limitation your commercialization of + any Application), breach of this Agreement or violation of the + rights of any other person or entity, except solely to the extent + any of the foregoing arise out of the willful misconduct or gross + negligence of Bytewax or an IP Claim for which Bytewax is obligated + under Section 15 below. Bytewax reserves the right, at your expense, + to assume the exclusive defense and control of any matter for which + you are required to indemnify the Bytewax and you agree to cooperate + with Bytewax's defense of these claims. + + 15. Subject to the terms and conditions hereof, Bytewax agrees to, at + its own expense, defend and/or settle any claim, action or suit + brought by a third party against you alleging that Software + infringes such third party's intellectual property rights (an "IP + Claim"), and Bytewax will pay those amounts finally awarded by a + court of competent jurisdiction against the Company Indemnitees, or + payable pursuant to a settlement agreed to by Bytewax with respect + to the IP Claim, provided that you promptly notify Bytewax in + writing of the IP Claim, tender sole control of the defense and + settlement thereof to Bytewax and provide Bytewax with all + reasonably requested information and assistance in connection with + the IP Claim. If Bytewax, in its sole discretion, believes an IP + Claim or an adverse judgment in connection with an IP Claim is + likely, then Bytewax may, at its option, (a) obtain a license from + such third party claimant that allows you to continue the use of the + Software, (b) modify the Software so as to be non-infringing, or (c) + terminate this License Agreement. Bytewax will have no obligation or + liability relating to any IP Claim that: (x) is based on + modification or customization of the Software by you or any person + or entity other than Bytewax; (y) is based on the combination or use + of the Software (or any component of either) with any software, + hardware, system, method, device or materials not provided by + Bytewax; or (z) results from your use of the Software in a manner + that is inconsistent with its intended use or is in breach of this + Agreement. This Section 15 sets forth the entire liability of + Bytewax and your sole and exclusive remedy in the event of any claim + that the Software infringes any third party intellectual property + right. + + 16. Bytewax may make modifications, deletions and/or additions to this + License Agreement ("Changes") at any time. Changes will be + effective: (i) thirty (30) days after Bytewax sends notice of the + Changes to the e-mail address associated with your account; or (ii) + when you opt-in or otherwise expressly agree to the Changes or a + version of this License Agreement incorporating the Changes, + whichever comes first. In addition, this License Agreement may be + modified by an amendment signed by authorized representatives of you + and Bytewax. This License Agreement may not be modified or amended + other than as set forth in this Section 16. + + 17. To the extent that you are an agency or instrumentality of the U.S. + government, the parties agree that the Software and Documentation + are commercial computer software and commercial computer software + documentation, respectively, and that your rights therein are as + specified in this License Agreement, per FAR 12.212 and DEARS + 227.7202-3, as applicable, or in the case of NASA, subject to NFS + 1852.227-86. + + 18. You may not assign your rights under this License Agreement without + the express prior consent of Bytewax. If you are a legal entity, any + merger involving you, acquisition of all or substantially all of + your assets or change of control shall be deemed an assignment of + this Agreement for which prior written consent is required. Bytewax + may freely assign this License Agreement. + + 19. The Software may be subject to export laws and regulations. You + agree to comply with any United States and international export laws + and regulations that may apply. + + 20. This License Agreement shall be governed by and construed in + accordance with the laws of the state of California, without giving + effect to any principles of conflicts of law. Any action arising out + of or relating to this License Agreement shall be filed only in the + state or federal courts of the Northern District of California and + you and Bytewax hereby consent and submit to the personal + jurisdiction of such courts for the purposes of litigating any such + action. + + 21. Notwithstanding anything to the contrary, Bytewax may apply to any + court of competent jurisdiction for injunctive or other equitable + relief. + + 22. This License Agreement and the User Agreement for Bytewax is the + complete and final agreement of the parties with respect to the + Software and Documentation. If any part of this License Agreement is + found to be void, unenforceable or invalid, that part will be deemed + stricken and will not affect the validity of the other provisions. + Failure by Bytewax to enforce any provision of this License + Agreement will not be deemed a waiver of future enforcement of that + or any other provision. + Project-URL: Homepage, https://github.com/bytewax/bytewax-clickhouse Project-URL: Bug Reports, https://github.com/bytewax/bytewax-clickhouse/issues/new Project-URL: Source, https://github.com/bytewax/bytewax-clickhouse/ @@ -54,6 +336,12 @@ ClickHouse Sink requires a PyArrow table, Schema and keys for order and partitio The Sink is eventually consistent based on the keys. +Add the import + +```python +from bytewax.clickhouse import operators as chop +``` + Add A schema and order by string to your code. ```python @@ -69,17 +357,32 @@ ORDER_BY = "metric, ts" define a pyarrow schema ```python -PA_SCHEMA = pa.schema([ - ('metric',pa.string()), - ('value',pa.float64()), - ('ts',pa.timestamp('us')), # microsecond - ]) +PA_SCHEMA = pa.schema( + [ + ("metric", pa.string()), + ("value", pa.float64()), + ("ts", pa.timestamp("us")), # microsecond + ] +) ``` Use the ClickHouse Sink to write data to ClickHouse ```python -op.output("output_clickhouse", tables, ClickhouseSink("metrics", "admin", "password", database="bytewax", port=8123, schema=CH_SCHEMA, order_by=ORDER_BY)) +chop.output( + "output_clickhouse", + metrics, + "metrics", + "admin", + "password", + database="bytewax", + port=8123, + ch_schema=CH_SCHEMA, + order_by=ORDER_BY, + pa_schema=PA_SCHEMA, + timeout=timedelta(seconds=1), + max_size=10, +) ``` ## Setting up the project