WIP: major refactor for q as a module

.github/workflows/docs.yaml

@@ -0,0 +1,37 @@
+name: Documentation
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - "mkdocs/**/*"
+      - "doc/**/*"
+      - "*.md"
+      - "*.markdown"
+  release:
+    types: [published]
+
+jobs:
+  deploy-docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install mkdocs mkdocs-material
+    - name: Build man page
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y ruby
+        gem install ronn
+        ronn doc/USAGE.markdown
+        gzip -c doc/USAGE > doc/USAGE.gz
+    - name: Deploy documentation
+      run: |
+        mkdocs build --config-file mkdocs/mkdocs.yml
+        mkdocs gh-deploy --force --config-file mkdocs/mkdocs.yml 

.github/workflows/publish-package.yaml

@@ -0,0 +1,59 @@
+name: Publish Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r test-requirements.txt
+        pip install -e .
+    - name: Run tests
+      run: |
+        pytest -v
+
+  deploy:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    - name: Update version from tag
+      run: |
+        VERSION=${GITHUB_REF#refs/tags/v}
+        echo "Setting version to $VERSION"
+        python3 ./bump-version.py --set "$VERSION"
+    - name: Verify package imports
+      run: |
+        python3 -c "from qtextasdata import QTextAsData, QInputParams, __version__; print('Version:', __version__)"
+    - name: Build package
+      run: |
+        python -m build
+        twine check dist/*
+    - name: Publish to PyPI
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      run: |
+        twine upload dist/*

.github/workflows/python-package.yaml

@@ -0,0 +1,55 @@
+name: Python Package
+
+on:
+  push:
+    branches: [ master ]
+    paths-ignore:
+      - "*.md"
+      - "*.markdown"
+      - "mkdocs/**/*"
+  pull_request:
+    branches: [ master ]
+    paths-ignore:
+      - "*.md"
+      - "*.markdown"
+      - "mkdocs/**/*"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install pytest pytest-cov
+        if [ -f test-requirements.txt ]; then pip install -r test-requirements.txt; fi
+        pip install -e .
+    - name: Test with pytest
+      run: |
+        pytest -v --cov=qtextasdata
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.9"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8
+    - name: Lint with flake8
+      run: |
+        flake8 qtextasdata --count --select=E9,F63,F7,F82 --show-source --statistics 

.gitignore

@@ -10,11 +10,11 @@ win_output
 win_build
 packages
 .idea/
-dist/windows/
+dist/
 generated-site/
 benchmark_data.tar.gz
 _benchmark_data/
-q.egg-info/
+*.egg-info/
 .pytest_cache/
 *.qsql
 htmlcov/

.python-version

@@ -0,0 +1 @@
+refactor-q

CHANGELOG.md

@@ -0,0 +1,64 @@
+# Changelog
+
+All notable changes to q (qtextasdata) will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Version bumping script (`bump-version.py`) for easier version management
+
+## [4.0.0] - (release date)
+
+Major release exposing q as a Python module under the package name `qtextasdata`.
+
+### BREAKING CHANGES
+- Package renamed from `q` to `qtextasdata`
+- Module structure completely refactored from a single `bin/q.py` file to a proper Python package (`qtextasdata/`)
+- CLI entry point changed from `bin.q:run_standalone` to `qtextasdata.cli:run_standalone`
+- Removed `six` dependency (Python 2 support fully dropped)
+- License updated to Apache License 2.0
+
+### Added
+- **Python module API** -- q can now be imported and used programmatically via `from qtextasdata import QTextAsData, QInputParams`
+- Public API classes exported from the top-level package: `QTextAsData`, `QInputParams`, `QOutput`, `QOutputPrinter`, `QMetadata`, `QWarning`, `QError`, `DataStream`, `DataStreams`
+- `__version__` attribute on the `qtextasdata` package
+- `__all__` definition for explicit public API surface
+- Data stream injection for in-memory data processing (`DataStream`, `data_streams_dict`)
+- Query-level data reuse across multiple `execute()` calls on the same engine
+- Pre-loading of data via `load_data()` method
+- Per-file input parameter support via `QInputParams.merged_with()`
+- Query analysis without execution via `analyze()` method
+- Dedicated module-level test suite (`test/BasicModuleTests.py`)
+- Comprehensive Python API reference documentation (`doc/PYTHON-API.md`)
+- GitHub Actions workflows for testing and documentation
+- `pytest.ini` configuration for test execution
+- Version bumping script (`bump-version.py`)
+- `CHANGELOG.md` following Keep a Changelog format
+
+### Changed
+- Codebase restructured into modular architecture:
+  - `qtextasdata/core.py` -- Engine, API classes, and output formatting
+  - `qtextasdata/cli.py` -- Command-line interface
+  - `qtextasdata/sql.py` -- SQL parsing, table materialization, and SQLite operations
+  - `qtextasdata/exceptions.py` -- Exception hierarchy
+  - `qtextasdata/utilities.py` -- Helper functions and user-defined SQL functions
+  - `qtextasdata/csv_reader.py` -- CSV parsing utilities
+  - `qtextasdata/logging.py` -- Debug logging
+- Test suite split from monolithic `test/test_suite.py` into focused test modules
+- `setup.py` updated with proper PyPI metadata, classifiers, and project URLs
+- README updated with Python module usage documentation and examples
+
+### Removed
+- `bin/q.py` monolithic single-file implementation (replaced by `qtextasdata/` package)
+- `bin/__init__.py` and `bin/.qrc` (package root moved out of `bin/`)
+- `bin/q.bat` Windows batch wrapper
+- `six` dependency
+- `pyoxidizer.bzl` build configuration
+- `requirements.txt` (dependencies declared in `setup.py`)
+- Old monolithic test suite `test/test_suite.py` (replaced by focused test modules)
+
+[Unreleased]: https://github.com/harelba/q/compare/v4.0.0...HEAD
+[4.0.0]: https://github.com/harelba/q/releases/tag/v4.0.0