Initial commit
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "26.4.10"
|
||||
commit = true
|
||||
commit_args = "--no-verify"
|
||||
tag = true
|
||||
tag_name = "v{new_version}"
|
||||
allow_dirty = false
|
||||
parse = """(?x) # Verbose mode
|
||||
(?P<release> # The release part
|
||||
(?:[1-9]{2})\\. # YY.
|
||||
(?:1[0-2]|[1-9])\\. # MM.
|
||||
(?:3[0-1]|[1-2][0-9]|[1-9]) # DD
|
||||
)
|
||||
(?:\\.(?P<patch>\\d+))? # .patch, optional
|
||||
"""
|
||||
serialize = ["{release}.{patch}", "{release}"]
|
||||
|
||||
[tool.bumpversion.parts.release]
|
||||
calver_format = "{YY}.{MM}.{DD}"
|
||||
|
||||
[[tool.bumpversion.files]]
|
||||
filename = "pixi.toml"
|
||||
search = 'version = "{current_version}"'
|
||||
replace = 'version = "{new_version}"'
|
||||
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
[run]
|
||||
branch = true
|
||||
data_file = .coverage
|
||||
source = "tests"
|
||||
|
||||
[paths]
|
||||
source = pcdigitizer
|
||||
|
||||
[report]
|
||||
show_missing = true
|
||||
skip_empty = true
|
||||
@@ -0,0 +1,80 @@
|
||||
# https://editorconfig.org
|
||||
root = true
|
||||
|
||||
# Global defaults
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
# Python
|
||||
[*.{py,pyi}]
|
||||
indent_size = 4
|
||||
max_line_length = 88
|
||||
|
||||
[*.ipynb]
|
||||
indent_size = 2
|
||||
|
||||
# Config / markup
|
||||
[*.{yml,yaml}]
|
||||
indent_size = 2
|
||||
|
||||
[*.{toml,cfg,ini}]
|
||||
indent_size = 4
|
||||
|
||||
[*.json]
|
||||
indent_size = 2
|
||||
|
||||
[*.{md,rst}]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
# Shell / scripts
|
||||
[*.{sh,bash,zsh}]
|
||||
indent_size = 2
|
||||
end_of_line = lf
|
||||
|
||||
[*.{bat,cmd,ps1}]
|
||||
end_of_line = crlf
|
||||
|
||||
# Web / docs
|
||||
[*.{html,htm,xml,css,js,ts}]
|
||||
indent_size = 2
|
||||
|
||||
[*.svg]
|
||||
indent_size = 2
|
||||
|
||||
# Structural biology: text formats
|
||||
[*.{pdb,ent,cif,mmcif}]
|
||||
indent_style = unset
|
||||
indent_size = unset
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
# Cheminformatics
|
||||
[*.{sdf,mol,mol2}]
|
||||
indent_style = unset
|
||||
indent_size = unset
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[*.{smi,sln,mae}]
|
||||
indent_size = 4
|
||||
|
||||
# Data files
|
||||
[*.{csv,tsv}]
|
||||
trim_trailing_whitespace = false
|
||||
indent_style = unset
|
||||
indent_size = unset
|
||||
|
||||
# Patches and special files
|
||||
[*.{diff,patch}]
|
||||
trim_trailing_whitespace = false
|
||||
end_of_line = unset
|
||||
|
||||
[{LICENSE,LICENSE.md,COPYING}]
|
||||
insert_final_newline = false
|
||||
|
||||
[*.{log,out}]
|
||||
trim_trailing_whitespace = false
|
||||
insert_final_newline = false
|
||||
@@ -0,0 +1,67 @@
|
||||
# Set the default behavior, in case people don't have core.autocrlf set.
|
||||
* text=auto
|
||||
|
||||
# Python source files
|
||||
*.py text eol=lf diff=python
|
||||
*.pyi text eol=lf diff=python
|
||||
*.ipynb text eol=lf
|
||||
|
||||
# Config / project files
|
||||
*.toml text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.ini text eol=lf
|
||||
*.env text eol=lf
|
||||
.gitignore text eol=lf
|
||||
.gitattributes text eol=lf
|
||||
|
||||
# Docs
|
||||
*.md text eol=lf diff=markdown
|
||||
*.rst text eol=lf
|
||||
|
||||
# Crystallography / structure deposition
|
||||
*.pdb text eol=lf
|
||||
*.cif text eol=lf
|
||||
*.mmcif text eol=lf
|
||||
*.ccp4 text eol=lf
|
||||
|
||||
# Cheminformatics
|
||||
*.sdf text eol=lf
|
||||
*.smi text eol=lf
|
||||
*.mol text eol=lf
|
||||
*.mol2 text eol=lf
|
||||
*.mae text eol=lf
|
||||
*.maegz binary
|
||||
*.sln text eol=lf
|
||||
|
||||
# General data formats
|
||||
*.parquet binary
|
||||
*.arrow binary
|
||||
*.feather binary
|
||||
*.pkl binary
|
||||
*.pickle binary
|
||||
*.npy binary
|
||||
*.npz binary
|
||||
*.csv text eol=lf
|
||||
*.tsv text eol=lf
|
||||
*.json text eol=lf
|
||||
*.jsonl text eol=lf
|
||||
|
||||
# Images
|
||||
*.png binary
|
||||
*.jpg binary
|
||||
*.jpeg binary
|
||||
*.gif binary
|
||||
*.svg text eol=lf
|
||||
*.ico binary
|
||||
*.webp binary
|
||||
*.tif binary
|
||||
*.tiff binary
|
||||
|
||||
# Linguist overrides
|
||||
pixi.lock linguist-language=YAML linguist-generated=true
|
||||
*.lock linguist-generated=true
|
||||
*.log linguist-detectable=false
|
||||
*.out linguist-detectable=false
|
||||
*.dlg linguist-detectable=false
|
||||
@@ -0,0 +1,51 @@
|
||||
name: Codecov
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'pcdigitizer/**'
|
||||
- 'tests/**'
|
||||
- 'pixi.toml'
|
||||
- 'pixi.lock'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'pcdigitizer/**'
|
||||
- 'tests/**'
|
||||
- 'pixi.toml'
|
||||
- 'pixi.lock'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
run:
|
||||
name: codecov
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
lfs: true
|
||||
|
||||
- name: Install pixi
|
||||
uses: prefix-dev/setup-pixi@v0.9.4
|
||||
with:
|
||||
environments: dev
|
||||
locked: false
|
||||
frozen: false
|
||||
cache: true
|
||||
cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
|
||||
activate-environment: true
|
||||
|
||||
- name: Get test coverage
|
||||
run: pixi run tests
|
||||
|
||||
- name: Upload to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
env_vars: OS,PYTHON
|
||||
fail_ci_if_error: true
|
||||
verbose: true
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
name: Documentation
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'pcdigitizer/**'
|
||||
- 'docs/**'
|
||||
- 'pixi.toml'
|
||||
- 'pixi.lock'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'pcdigitizer/**'
|
||||
- 'docs/**'
|
||||
- 'pixi.toml'
|
||||
- 'pixi.lock'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: "pages"
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: docs
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install pixi
|
||||
uses: prefix-dev/setup-pixi@v0.9.4
|
||||
with:
|
||||
environments: docs
|
||||
locked: false
|
||||
frozen: false
|
||||
cache: true
|
||||
cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
|
||||
activate-environment: true
|
||||
|
||||
- name: Build documentation
|
||||
run: pixi run docs
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v5
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v4
|
||||
with:
|
||||
path: 'public/'
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
@@ -0,0 +1,39 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'pcdigitizer/**'
|
||||
- 'tests/**'
|
||||
- 'pixi.toml'
|
||||
- 'pixi.lock'
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
paths:
|
||||
- 'pcdigitizer/**'
|
||||
- 'tests/**'
|
||||
- 'pixi.toml'
|
||||
- 'pixi.lock'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install pixi
|
||||
uses: prefix-dev/setup-pixi@v0.9.4
|
||||
with:
|
||||
environments: dev
|
||||
locked: false
|
||||
frozen: false
|
||||
cache: true
|
||||
cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
|
||||
activate-environment: true
|
||||
|
||||
- name: Run tests
|
||||
run: pixi run tests
|
||||
+752
@@ -0,0 +1,752 @@
|
||||
# pixi environments
|
||||
.pixi/*
|
||||
!.pixi/config.toml
|
||||
*/_version.py
|
||||
|
||||
# Jupyter Notebook
|
||||
**/.ipynb_checkpoints
|
||||
|
||||
# Coverage
|
||||
report.xml
|
||||
|
||||
|
||||
# Created by https://www.toptal.com/developers/gitignore/api/osx,python,pycharm,windows,visualstudio,visualstudiocode,zsh,jetbrains,jupyternotebooks
|
||||
# Edit at https://www.toptal.com/developers/gitignore?templates=osx,python,pycharm,windows,visualstudio,visualstudiocode,zsh,jetbrains,jupyternotebooks
|
||||
|
||||
### JetBrains ###
|
||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||
|
||||
# User-specific stuff
|
||||
.idea/**/workspace.xml
|
||||
.idea/**/tasks.xml
|
||||
.idea/**/usage.statistics.xml
|
||||
.idea/**/dictionaries
|
||||
.idea/**/shelf
|
||||
|
||||
# AWS User-specific
|
||||
.idea/**/aws.xml
|
||||
|
||||
# Generated files
|
||||
.idea/**/contentModel.xml
|
||||
|
||||
# Sensitive or high-churn files
|
||||
.idea/**/dataSources/
|
||||
.idea/**/dataSources.ids
|
||||
.idea/**/dataSources.local.xml
|
||||
.idea/**/sqlDataSources.xml
|
||||
.idea/**/dynamic.xml
|
||||
.idea/**/uiDesigner.xml
|
||||
.idea/**/dbnavigator.xml
|
||||
|
||||
# Gradle
|
||||
.idea/**/gradle.xml
|
||||
.idea/**/libraries
|
||||
|
||||
# CMake
|
||||
cmake-build-*/
|
||||
|
||||
# Mongo Explorer plugin
|
||||
.idea/**/mongoSettings.xml
|
||||
|
||||
# File-based project format
|
||||
*.iws
|
||||
|
||||
# IntelliJ
|
||||
out/
|
||||
|
||||
# mpeltonen/sbt-idea plugin
|
||||
.idea_modules/
|
||||
|
||||
# JIRA plugin
|
||||
atlassian-ide-plugin.xml
|
||||
|
||||
# Cursive Clojure plugin
|
||||
.idea/replstate.xml
|
||||
|
||||
# SonarLint plugin
|
||||
.idea/sonarlint/
|
||||
|
||||
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
|
||||
# Editor-based Rest Client
|
||||
.idea/httpRequests
|
||||
|
||||
# Android studio 3.1+ serialized cache file
|
||||
.idea/caches/build_file_checksums.ser
|
||||
|
||||
### JetBrains Patch ###
|
||||
|
||||
# Sonarlint plugin
|
||||
# https://plugins.jetbrains.com/plugin/7973-sonarlint
|
||||
.idea/**/sonarlint/
|
||||
|
||||
# SonarQube Plugin
|
||||
# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
|
||||
.idea/**/sonarIssues.xml
|
||||
|
||||
# Markdown Navigator plugin
|
||||
# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
|
||||
.idea/**/markdown-navigator.xml
|
||||
.idea/**/markdown-navigator-enh.xml
|
||||
.idea/**/markdown-navigator/
|
||||
|
||||
# Cache file creation bug
|
||||
# See https://youtrack.jetbrains.com/issue/JBR-2257
|
||||
.idea/$CACHE_FILE$
|
||||
|
||||
# CodeStream plugin
|
||||
# https://plugins.jetbrains.com/plugin/12206-codestream
|
||||
.idea/codestream.xml
|
||||
|
||||
# Azure Toolkit for IntelliJ plugin
|
||||
# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
|
||||
.idea/**/azureSettings.xml
|
||||
|
||||
### JupyterNotebooks ###
|
||||
# gitignore template for Jupyter Notebooks
|
||||
# website: http://jupyter.org/
|
||||
|
||||
.ipynb_checkpoints
|
||||
*/.ipynb_checkpoints/*
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
### OSX ###
|
||||
# General
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
.LSOverride
|
||||
|
||||
# Icon must end with two \r
|
||||
Icon
|
||||
|
||||
|
||||
# Thumbnails
|
||||
._*
|
||||
|
||||
# Files that might appear in the root of a volume
|
||||
.DocumentRevisions-V100
|
||||
.fseventsd
|
||||
.Spotlight-V100
|
||||
.TemporaryItems
|
||||
.Trashes
|
||||
.VolumeIcon.icns
|
||||
.com.apple.timemachine.donotpresent
|
||||
|
||||
# Directories potentially created on remote AFP share
|
||||
.AppleDB
|
||||
.AppleDesktop
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
### Python ###
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/#use-with-ide
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
### Python Patch ###
|
||||
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
|
||||
poetry.toml
|
||||
|
||||
# ruff
|
||||
.ruff_cache/
|
||||
|
||||
# LSP config files
|
||||
pyrightconfig.json
|
||||
|
||||
### VisualStudioCode ###
|
||||
.vscode/*
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
# Built Visual Studio Code Extensions
|
||||
*.vsix
|
||||
|
||||
### VisualStudioCode Patch ###
|
||||
# Ignore all local history of files
|
||||
.history
|
||||
.ionide
|
||||
|
||||
### Windows ###
|
||||
# Windows thumbnail cache files
|
||||
Thumbs.db
|
||||
Thumbs.db:encryptable
|
||||
ehthumbs.db
|
||||
ehthumbs_vista.db
|
||||
|
||||
# Dump file
|
||||
*.stackdump
|
||||
|
||||
# Folder config file
|
||||
[Dd]esktop.ini
|
||||
|
||||
# Recycle Bin used on file shares
|
||||
$RECYCLE.BIN/
|
||||
|
||||
# Windows Installer files
|
||||
*.cab
|
||||
*.msi
|
||||
*.msix
|
||||
*.msm
|
||||
*.msp
|
||||
|
||||
# Windows shortcuts
|
||||
*.lnk
|
||||
|
||||
### Zsh ###
|
||||
# Zsh compiled script + zrecompile backup
|
||||
*.zwc
|
||||
*.zwc.old
|
||||
|
||||
# Zsh completion-optimization dumpfile
|
||||
*zcompdump*
|
||||
|
||||
# Zsh history
|
||||
.zsh_history
|
||||
|
||||
# Zsh sessions
|
||||
.zsh_sessions
|
||||
|
||||
# Zsh zcalc history
|
||||
.zcalc_history
|
||||
|
||||
# A popular plugin manager's files
|
||||
._zinit
|
||||
.zinit_lstupd
|
||||
|
||||
# zdharma/zshelldoc tool's files
|
||||
zsdoc/data
|
||||
|
||||
# robbyrussell/oh-my-zsh/plugins/per-directory-history plugin's files
|
||||
# (when set-up to store the history in the local directory)
|
||||
.directory_history
|
||||
|
||||
# MichaelAquilina/zsh-autoswitch-virtualenv plugin's files
|
||||
# (for Zsh plugins using Python)
|
||||
|
||||
# Zunit tests' output
|
||||
/tests/_output/*
|
||||
!/tests/_output/.gitkeep
|
||||
|
||||
### VisualStudio ###
|
||||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore
|
||||
|
||||
# User-specific files
|
||||
*.rsuser
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Mono auto generated files
|
||||
mono_crash.*
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
[Ww][Ii][Nn]32/
|
||||
[Aa][Rr][Mm]/
|
||||
[Aa][Rr][Mm]64/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
[Ll]ogs/
|
||||
|
||||
# Visual Studio 2015/2017 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# Visual Studio 2017 auto generated files
|
||||
Generated\ Files/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUnit
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
nunit-*.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# Benchmark Results
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# .NET Core
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
|
||||
# ASP.NET Scaffolding
|
||||
ScaffoldingReadMe.txt
|
||||
|
||||
# StyleCop
|
||||
StyleCopReport.xml
|
||||
|
||||
# Files built by Visual Studio
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_h.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.iobj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.ipdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*_wpftmp.csproj
|
||||
*.tlog
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# Visual Studio Trace Files
|
||||
*.e2e
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# AxoCover is a Code Coverage Tool
|
||||
.axoCover/*
|
||||
!.axoCover/settings.json
|
||||
|
||||
# Coverlet is a free, cross platform Code Coverage Tool
|
||||
coverage*.json
|
||||
coverage*.xml
|
||||
coverage*.info
|
||||
|
||||
# Visual Studio code coverage results
|
||||
*.coverage
|
||||
*.coveragexml
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# NuGet Symbol Packages
|
||||
*.snupkg
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/[Pp]ackages/*
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/[Pp]ackages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/[Pp]ackages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignorable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
*.appx
|
||||
*.appxbundle
|
||||
*.appxupload
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!?*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
orleans.codegen.cs
|
||||
|
||||
# Including strong name files can present a security risk
|
||||
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||
#*.snk
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
ServiceFabricBackup/
|
||||
*.rptproj.bak
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
*.ndf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
*.rptproj.rsuser
|
||||
*- [Bb]ackup.rdl
|
||||
*- [Bb]ackup ([0-9]).rdl
|
||||
*- [Bb]ackup ([0-9][0-9]).rdl
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
node_modules/
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||
*.vbw
|
||||
|
||||
# Visual Studio 6 auto-generated project file (contains which files were open etc.)
|
||||
*.vbp
|
||||
|
||||
# Visual Studio 6 workspace and project file (working project files containing files to include in project)
|
||||
*.dsw
|
||||
*.dsp
|
||||
|
||||
# Visual Studio 6 technical files
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# CodeRush personal settings
|
||||
.cr/personal
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
*.pyc
|
||||
|
||||
# Cake - Uncomment if you are using it
|
||||
# tools/**
|
||||
# !tools/packages.config
|
||||
|
||||
# Tabs Studio
|
||||
*.tss
|
||||
|
||||
# Telerik's JustMock configuration file
|
||||
*.jmconfig
|
||||
|
||||
# BizTalk build output
|
||||
*.btp.cs
|
||||
*.btm.cs
|
||||
*.odx.cs
|
||||
*.xsd.cs
|
||||
|
||||
# OpenCover UI analysis results
|
||||
OpenCover/
|
||||
|
||||
# Azure Stream Analytics local run output
|
||||
ASALocalRun/
|
||||
|
||||
# MSBuild Binary and Structured Log
|
||||
*.binlog
|
||||
|
||||
# NVidia Nsight GPU debugger configuration file
|
||||
*.nvuser
|
||||
|
||||
# MFractors (Xamarin productivity tool) working folder
|
||||
.mfractor/
|
||||
|
||||
# Local History for Visual Studio
|
||||
.localhistory/
|
||||
|
||||
# Visual Studio History (VSHistory) files
|
||||
.vshistory/
|
||||
|
||||
# BeatPulse healthcheck temp database
|
||||
healthchecksdb
|
||||
|
||||
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
||||
MigrationBackup/
|
||||
|
||||
# Ionide (cross platform F# VS Code tools) working folder
|
||||
.ionide/
|
||||
|
||||
# Fody - auto-generated XML schema
|
||||
FodyWeavers.xsd
|
||||
|
||||
# VS Code files for those working on multiple tools
|
||||
*.code-workspace
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
|
||||
# Windows Installer files from build outputs
|
||||
|
||||
# JetBrains Rider
|
||||
*.sln.iml
|
||||
|
||||
### VisualStudio Patch ###
|
||||
# Additional files built by Visual Studio
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/osx,python,pycharm,windows,visualstudio,visualstudiocode,zsh,jetbrains,jupyternotebooks
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
[pytest]
|
||||
norecursedirs =
|
||||
pcdigitizer
|
||||
*.egg
|
||||
.eggs
|
||||
dist
|
||||
build
|
||||
docs
|
||||
.tox
|
||||
.git
|
||||
__pycache__
|
||||
doctest_optionflags =
|
||||
NUMBER
|
||||
NORMALIZE_WHITESPACE
|
||||
IGNORE_EXCEPTION_DETAIL
|
||||
addopts =
|
||||
--strict-markers
|
||||
--tb=short
|
||||
--doctest-modules
|
||||
--doctest-continue-on-failure
|
||||
--maxfail=2 -rf
|
||||
testpaths = tests
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
target-version = "py313"
|
||||
line-length = 88
|
||||
indent-width = 4
|
||||
exclude = [
|
||||
".bzr",
|
||||
".direnv",
|
||||
".eggs",
|
||||
".git",
|
||||
".git-rewrite",
|
||||
".hg",
|
||||
".ipynb_checkpoints",
|
||||
".mypy_cache",
|
||||
".nox",
|
||||
".pants.d",
|
||||
".pyenv",
|
||||
".pytest_cache",
|
||||
".pytype",
|
||||
".ruff_cache",
|
||||
".svn",
|
||||
".tox",
|
||||
".venv",
|
||||
".vscode",
|
||||
"__pypackages__",
|
||||
"_build",
|
||||
"buck-out",
|
||||
"build",
|
||||
"dist",
|
||||
"node_modules",
|
||||
"site-packages",
|
||||
"venv",
|
||||
".pixi",
|
||||
"__init__.py"
|
||||
]
|
||||
|
||||
[lint]
|
||||
preview = true
|
||||
select = ["E1", "E2", "E3", "E4", "E7", "W1", "W2", "W3", "W5", "W6", "F"]
|
||||
fixable = ["ALL"]
|
||||
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
||||
|
||||
[format]
|
||||
quote-style = "double"
|
||||
indent-style = "space"
|
||||
skip-magic-trailing-comma = false
|
||||
line-ending = "auto"
|
||||
docstring-code-format = true
|
||||
docstring-code-line-length = "dynamic"
|
||||
@@ -0,0 +1,7 @@
|
||||
# Changelog
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Calendar Versioning](https://calver.org/).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
- Initial release!
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
# The Prosperity Public License 3.0.0
|
||||
|
||||
Contributor: Scientific Computing Studio
|
||||
|
||||
Source Code: <https://github.com/scienting/pcdigitizer>
|
||||
|
||||
## Purpose
|
||||
|
||||
This license allows you to use and share this software for noncommercial purposes for free and to try this software for commercial purposes for thirty days.
|
||||
|
||||
## Agreement
|
||||
|
||||
In order to receive this license, you have to agree to its rules.
|
||||
Those rules are both obligations under that agreement and conditions to your license.
|
||||
Don't do anything with this software that triggers a rule you can't or won't follow.
|
||||
|
||||
## Notices
|
||||
|
||||
Make sure everyone who gets a copy of any part of this software from you, with or without changes, also gets the text of this license and the contributor and source code lines above.
|
||||
|
||||
## Commercial Trial
|
||||
|
||||
Limit your use of this software for commercial purposes to a thirty-day trial period.
|
||||
If you use this software for work, your company gets one trial period for all personnel, not one trial per person.
|
||||
|
||||
## Contributions Back
|
||||
|
||||
Developing feedback, changes, or additions that you contribute back to the contributor on the terms of a standardized public software license such as [the Blue Oak Model License 1.0.0](https://blueoakcouncil.org/license/1.0.0), [the Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html), [the MIT license](https://spdx.org/licenses/MIT.html), or [the two-clause BSD license](https://spdx.org/licenses/BSD-2-Clause.html) doesn't count as use for a commercial purpose.
|
||||
|
||||
## Personal Uses
|
||||
|
||||
Personal use for research, experiment, and testing for the benefit of public knowledge, personal study, private entertainment, hobby projects, amateur pursuits, or religious observance, without any anticipated commercial application, doesn't count as use for a commercial purpose.
|
||||
|
||||
## Noncommercial Organizations
|
||||
|
||||
Use by any charitable organization, educational institution, public research organization, public safety or health organization, environmental protection organization, or government institution doesn't count as use for a commercial purpose regardless of the source of funding or obligations resulting from the funding.
|
||||
|
||||
## Defense
|
||||
|
||||
Don't make any legal claim against anyone accusing this software, with or without changes, alone or with other technology, of infringing any patent.
|
||||
|
||||
## Copyright
|
||||
|
||||
The contributor licenses you to do everything with this software that would otherwise infringe their copyright in it.
|
||||
|
||||
## Patent
|
||||
|
||||
The contributor licenses you to do everything with this software that would otherwise infringe any patents they can license or become able to license.
|
||||
|
||||
## Reliability
|
||||
|
||||
The contributor can't revoke this license.
|
||||
|
||||
## Excuse
|
||||
|
||||
You're excused for unknowingly breaking [Notices](#notices) if you take all practical steps to comply within thirty days of learning you broke the rule.
|
||||
|
||||
## No Liability
|
||||
|
||||
***As far as the law allows, this software comes as is, without any warranty or condition, and the contributor won't be liable to anyone for any damages related to this software or this license, under any kind of legal claim.***
|
||||
@@ -0,0 +1,92 @@
|
||||
<h1 align="center">pcdigitizer</h1>
|
||||
|
||||
<h4 align="center">Turn raw PubChem data into clean, ML-ready chemical datasets</h4>
|
||||
|
||||
`pcdigitizer` is a Python toolkit for building ML-ready chemical datasets from [PubChem](https://pubchem.ncbi.nlm.nih.gov/).
|
||||
It handles the full pipeline: downloading raw data through PubChem's [PUG-REST API](https://pubchem.ncbi.nlm.nih.gov/docs/pug-rest), parsing nested and inconsistent response formats, and cleaning the results into structured [Polars](https://pola.rs/) DataFrames ready for analysis or model training.
|
||||
|
||||
PubChem is the largest open chemical database in the world, but its data comes from thousands of depositors with varying formats, conventions, and quality levels.
|
||||
Turning that into something you can feed to a machine learning model takes substantial data engineering.
|
||||
`pcdigitizer` does that work for you.
|
||||
|
||||
> [!CAUTION]
|
||||
> `pcdigitizer` is under active development and not yet ready for production use.
|
||||
> APIs, data formats, and cleaning methodology may change without notice.
|
||||
|
||||
## Installation
|
||||
|
||||
Install directly from GitHub:
|
||||
|
||||
```bash
|
||||
pip install git+https://github.com/scienting/pcdigitizer.git
|
||||
```
|
||||
|
||||
Or clone and install locally:
|
||||
|
||||
```bash
|
||||
git clone git@github.com:scienting/pcdigitizer.git
|
||||
cd pcdigitizer
|
||||
pip install .
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
`pcdigitizer` can also download and parse specific annotation headings into structured polars DataFrames.
|
||||
Use the `Annotation` enum to select a supported annotation and `GetAnnotationPage` to fetch and process the first page of results.
|
||||
|
||||
```python
|
||||
from pcdigitizer import Annotation, GetAnnotationPage
|
||||
|
||||
df = GetAnnotationPage().do(item=1, annotation=Annotation.DISSOCIATION_CONSTANTS)
|
||||
```
|
||||
|
||||
This downloads the first page of dissociation constant data from PubChem, parses the free-text pKa strings, and returns a DataFrame with columns for `cid`, `sid`, `pka_value`, `temperature_C`, and more.
|
||||
For example, here are the first five columns from the example above.
|
||||
|
||||
```text
|
||||
shape: (5, 7)
|
||||
┌──────┬─────┬───────────┬───────────┬───────────┬───────────────┬─────────────────────────────────┐
|
||||
│ cid ┆ sid ┆ pclid ┆ pka_label ┆ pka_value ┆ temperature_C ┆ comment │
|
||||
│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │
|
||||
│ i64 ┆ i64 ┆ i64 ┆ str ┆ f64 ┆ f64 ┆ str │
|
||||
╞══════╪═════╪═══════════╪═══════════╪═══════════╪═══════════════╪═════════════════════════════════╡
|
||||
│ 2519 ┆ 36 ┆ 900032672 ┆ pKa ┆ 14.0 ┆ 25.0 ┆ pKa = 14.0 at 25 °C │
|
||||
│ 2519 ┆ 36 ┆ 906276393 ┆ pKa ┆ 10.4 ┆ 40.0 ┆ pKa = 10.4 at 40 °C (tertiary … │
|
||||
│ 2519 ┆ 36 ┆ 900025588 ┆ pKa ┆ 0.7 ┆ null ┆ pKa = 0.7 (caffeine cation) │
|
||||
│ 176 ┆ 40 ┆ 900084372 ┆ pKa ┆ 4.76 ┆ 25.0 ┆ pKa = 4.76 at 25 °C │
|
||||
│ 180 ┆ 41 ┆ 900027470 ┆ pKa ┆ 20.0 ┆ null ┆ pKa = 20 │
|
||||
└──────┴─────┴───────────┴───────────┴───────────┴───────────────┴─────────────────────────────────┘
|
||||
```
|
||||
|
||||
`pcdigitizer` (will eventually) supports lookups across compounds, substances, assays, genes, proteins, pathways, taxonomies, cells, and annotations.
|
||||
You can query by name, identifier, structure (SMILES, InChI, InChIKey), formula, or accession number depending on the domain.
|
||||
See the [API documentation](https://scienting.github.io/pcdigitizer/) for the full reference.
|
||||
|
||||
## Development
|
||||
|
||||
We use [Pixi](https://pixi.sh/latest/) to manage environments and dependencies.
|
||||
After installing [Pixi](https://pixi.sh/latest/), clone the repo and run:
|
||||
|
||||
```bash
|
||||
pixi install
|
||||
pixi shell -e dev
|
||||
```
|
||||
|
||||
This gives you a fully configured environment with testing, linting, formatting, and build tools.
|
||||
See the [development guide](https://scienting.github.io/pcdigitizer/development) for the complete walkthrough, including how to run tests, build the package, and publish releases.
|
||||
|
||||
## License
|
||||
|
||||
`pcdigitizer` is licensed under the [Prosperity Public License 3.0.0](https://github.com/scienting/pcdigitizer/blob/main/LICENSE.md).
|
||||
|
||||
**Noncommercial use is free.**
|
||||
Academic researchers, university labs, nonprofits, and individual learners can use, modify, and distribute this software and any datasets it generates at no cost.
|
||||
If you use `pcdigitizer` in published research, a citation is appreciated.
|
||||
|
||||
**Commercial use requires a paid license.**
|
||||
The Prosperity Public License includes a 30-day trial period for commercial evaluation.
|
||||
After that, for-profit companies using `pcdigitizer` or its outputs in commercial products, services, or internal operations need a commercial license.
|
||||
This includes using the generated datasets in proprietary ML pipelines, commercial drug discovery workflows, or products built on the cleaned data.
|
||||
|
||||
Revenue from commercial licenses funds continued development, data validation, and maintenance of this project.
|
||||
To purchase a commercial license, contact [us@scient.ing](mailto:us@scient.ing).
|
||||
@@ -0,0 +1,10 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block header %}
|
||||
{{ super() }}
|
||||
<script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||||
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@24,400,0,0" />
|
||||
{% endblock %}
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
|
||||
/*Make the content wider and relative to window size.*/
|
||||
.md-grid {
|
||||
max-width: 85%
|
||||
}
|
||||
|
||||
:root {
|
||||
--md-tooltip-width: 600px;
|
||||
}
|
||||
|
||||
@page {
|
||||
size: letter;
|
||||
max-width: 100%;
|
||||
margin-top: 1in;
|
||||
margin-right: 0in;
|
||||
margin-bottom: 1in;
|
||||
margin-left: 0in;
|
||||
}
|
||||
|
||||
.md-typeset h2 {
|
||||
line-height: 1.13;
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
[data-md-color-scheme="default"] {
|
||||
|
||||
/* Primary color shades */
|
||||
--md-primary-fg-color: #025099;
|
||||
--md-primary-fg-color--light: #0437AD;
|
||||
--md-primary-bg-color: #ffffff; /* Header text */
|
||||
--md-primary-bg-color--light: #DBDBDB; /* Secondary header text */
|
||||
|
||||
/* Default color shades */
|
||||
--md-default-fg-color: #646464; /* ??? */
|
||||
--md-default-fg-color--light: #7A7A7A; /* h1 */
|
||||
--md-default-fg-color--lighter: #9B9B9B; /* ??? */
|
||||
--md-default-fg-color--lightest: #BCBCBC; /* ??? */
|
||||
|
||||
--md-default-bg-color: #FAFAFA; /* Body background */
|
||||
--md-default-bg-color--light: #FAFAFA;
|
||||
--md-default-bg-color--lighter: #FAFAFA;
|
||||
--md-default-bg-color--lightest: #FAFAFA;
|
||||
|
||||
/* Code color shades */
|
||||
--md-code-fg-color: #36464e; /* Code block text color */
|
||||
--md-code-bg-color: #f1f1f1; /* Code block background */
|
||||
|
||||
/* Code highlighting color shades */
|
||||
--md-code-hl-color: #0000ff;
|
||||
--md-code-hl-color--light: #0000ff;
|
||||
--md-code-hl-number-color: #d52a2a;
|
||||
--md-code-hl-special-color: #db1457;
|
||||
--md-code-hl-function-color: #a846b9;
|
||||
--md-code-hl-constant-color: #6e59d9;
|
||||
--md-code-hl-keyword-color: #3f6ec6;
|
||||
--md-code-hl-string-color: #1c7d4d;
|
||||
--md-code-hl-name-color: #36464e;
|
||||
--md-code-hl-operator-color: var(--md-primary-fg-color);
|
||||
--md-code-hl-punctuation-color: var(--md-primary-fg-color);
|
||||
--md-code-hl-comment-color: var(--md-primary-fg-color);
|
||||
--md-code-hl-generic-color: var(--md-primary-fg-color);
|
||||
--md-code-hl-variable-color: var(--md-primary-fg-color);
|
||||
|
||||
/* Typeset color shades */
|
||||
--md-typeset-color: #212529; /* Main text color */
|
||||
|
||||
/* Typeset `a` color shades */
|
||||
--md-typeset-a-color: #01a0d7; /* Link color */
|
||||
|
||||
/* Typeset `table` color shades */
|
||||
--md-typeset-table-color: #a5a5a5; /* Outline color */
|
||||
--md-typeset-table-color--light: #e3e2e2; /* Hover color */
|
||||
|
||||
/* Footer color shades */
|
||||
--md-footer-fg-color: #ffffff; /* ??? */
|
||||
--md-footer-fg-color--light: #e9ecef; /* Footer text */
|
||||
--md-footer-fg-color--lighter: #adb5bd; /* ??? */
|
||||
--md-footer-bg-color: #000000;
|
||||
--md-footer-bg-color--dark: #212529; /* Footer background */
|
||||
|
||||
/* Accent color shades */
|
||||
--md-accent-fg-color: #032779; /* Hover over link */
|
||||
--md-accent-fg-color--transparent: #caf0f8; /* Hover over transparent (e.g., code with link) */
|
||||
--md-accent-bg-color: #ffffff;
|
||||
--md-accent-bg-color--light: #e5e5e5;
|
||||
|
||||
/* Admonition colors */
|
||||
--md-admonition-fg-color: #212529;
|
||||
--md-admonition-bg-color: #FAFAFA;
|
||||
}
|
||||
|
||||
[data-md-color-scheme="dark"] {
|
||||
|
||||
/* Primary color shades */
|
||||
--md-primary-fg-color: #23243D;
|
||||
--md-primary-fg-color--light: #0437AD;
|
||||
--md-primary-bg-color: #ffffff; /* Header text */
|
||||
--md-primary-bg-color--light: #DBDBDB; /* Secondary header text */
|
||||
|
||||
/* Default color shades */
|
||||
--md-default-fg-color: #e2e4e9; /* ??? */
|
||||
--md-default-fg-color--light: #ffffff; /* h1 */
|
||||
--md-default-fg-color--lighter: #e2e4e9; /* ??? */
|
||||
--md-default-fg-color--lightest: #e2e4e9; /* ??? */
|
||||
|
||||
--md-default-bg-color: #212529; /* Body background */
|
||||
--md-default-bg-color--light: #FAFAFA;
|
||||
--md-default-bg-color--lighter: #FAFAFA;
|
||||
--md-default-bg-color--lightest: #FAFAFA;
|
||||
|
||||
/* Code color shades */
|
||||
--md-code-fg-color: #dddddd; /* Code block text color */
|
||||
--md-code-bg-color: #333333; /* Code block background */
|
||||
|
||||
/* Code highlighting color shades */
|
||||
--md-code-hl-color: #aeaeff;
|
||||
--md-code-hl-color--light: #aeaeff;
|
||||
--md-code-hl-number-color: #ff9494;
|
||||
--md-code-hl-special-color: #ffa0c0;
|
||||
--md-code-hl-function-color: #f3adff;
|
||||
--md-code-hl-constant-color: #bdaeff;
|
||||
--md-code-hl-keyword-color: #a0c1ff;
|
||||
--md-code-hl-string-color: #9fffcf;
|
||||
--md-code-hl-name-color: #f5f5f5;
|
||||
--md-code-hl-operator-color: #a6f0ff;
|
||||
--md-code-hl-punctuation-color: #a6f0ff;
|
||||
--md-code-hl-comment-color: #a6f0ff;
|
||||
--md-code-hl-generic-color: #a6f0ff;
|
||||
--md-code-hl-variable-color: #a6f0ff;
|
||||
|
||||
/* Typeset color shades */
|
||||
--md-typeset-color: #ffffff; /* Main text color */
|
||||
|
||||
/* Typeset `a` color shades */
|
||||
--md-typeset-a-color: #96E4FE; /* Link color */
|
||||
|
||||
/* Typeset `table` color shades */
|
||||
--md-typeset-table-color: #a5a5a5; /* Outline color */
|
||||
--md-typeset-table-color--light: #343a40; /* Hover color */
|
||||
|
||||
/* Footer color shades */
|
||||
--md-footer-fg-color: #ffffff; /* ??? */
|
||||
--md-footer-fg-color--light: #e9ecef; /* Footer text */
|
||||
--md-footer-fg-color--lighter: #adb5bd; /* ??? */
|
||||
--md-footer-bg-color: #000000;
|
||||
--md-footer-bg-color--dark: #171717; /* Footer background */
|
||||
|
||||
/* Accent color shades */
|
||||
--md-accent-fg-color: #90e0ef; /* Hover over link */
|
||||
--md-accent-fg-color--transparent: #6D6D6D; /* Hover over transparent (e.g., code with link) */
|
||||
--md-accent-bg-color: #ffffff;
|
||||
--md-accent-bg-color--light: #e5e5e5;
|
||||
|
||||
/* Admonition colors */
|
||||
--md-admonition-fg-color: #ffffff;
|
||||
--md-admonition-bg-color: #212529;
|
||||
|
||||
.highlight-ipynb {
|
||||
--jp-mirror-editor-string-color: #98c379;
|
||||
--jp-mirror-editor-number-color: #d19a66;
|
||||
--jp-mirror-editor-keyword-color: #c678dd;
|
||||
--jp-mirror-editor-operator-color: #c678dd;
|
||||
}
|
||||
|
||||
|
||||
|
||||
.highlight-ipynb {
|
||||
margin: 0;
|
||||
padding: 5px 10px;
|
||||
background-color: #23262A;
|
||||
}
|
||||
|
||||
.highlight-ipynb .nf {
|
||||
color: #61afef;
|
||||
}
|
||||
|
||||
.highlight-ipynb .p {
|
||||
color: #ffffff;
|
||||
}
|
||||
|
||||
.highlight-ipynb .nb {
|
||||
color: #56b6c2;
|
||||
}
|
||||
|
||||
.highlight-ipynb .kc {
|
||||
color: #d19a66;
|
||||
}
|
||||
|
||||
.highlight-ipynb .c1 {
|
||||
color: #8f8f8f;
|
||||
}
|
||||
|
||||
.jupyter-wrapper .jp-InputArea-editor {
|
||||
position: relative;
|
||||
border-color: #30363C;
|
||||
}
|
||||
|
||||
.jupyter-wrapper .highlight pre {
|
||||
background-color: transparent;
|
||||
padding: 10px;
|
||||
overflow: auto;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
html
|
||||
/*
|
||||
This adjusts the font size of Jupyter notebook code blocks to be closer to normal.
|
||||
*/
|
||||
.highlight {
|
||||
font-size: 85%;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
.launchy-container {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
position: relative;
|
||||
margin-top: -4.4em;
|
||||
margin-bottom: 2em;
|
||||
margin-right: 0.4em;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*Indentation.*/
|
||||
div.doc-contents:not(.first) {
|
||||
padding-left: 35px; /*25px is the default*/
|
||||
border-left: .15rem solid #ededed;
|
||||
}
|
||||
|
||||
.doc-heading .highlight {
|
||||
font-size: 18px;
|
||||
background-color: transparent;
|
||||
}
|
||||
|
||||
/*Mark external links as such. */
|
||||
a.external::after,
|
||||
a.autorefs-external::after {
|
||||
/* <https://primer.style/octicons/arrow-up-right-24>*/
|
||||
mask-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18.25 15.5a.75.75 0 00.75-.75v-9a.75.75 0 00-.75-.75h-9a.75.75 0 000 1.5h7.19L6.22 16.72a.75.75 0 101.06 1.06L17.5 7.56v7.19c0 .414.336.75.75.75z"></path></svg>');
|
||||
-webkit-mask-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18.25 15.5a.75.75 0 00.75-.75v-9a.75.75 0 00-.75-.75h-9a.75.75 0 000 1.5h7.19L6.22 16.72a.75.75 0 101.06 1.06L17.5 7.56v7.19c0 .414.336.75.75.75z"></path></svg>');
|
||||
content: ' ';
|
||||
|
||||
display: inline-block;
|
||||
vertical-align: middle;
|
||||
position: relative;
|
||||
|
||||
height: 1em;
|
||||
width: 1em;
|
||||
background-color: var(--md-typeset-a-color);
|
||||
}
|
||||
|
||||
a.external:hover::after,
|
||||
a.autorefs-external:hover::after {
|
||||
background-color: var(--md-accent-fg-color);
|
||||
}
|
||||
|
||||
/* Fancier color for operators such as * and |. */
|
||||
.doc-signature .o {
|
||||
color: var(--md-code-hl-special-color);
|
||||
}
|
||||
|
||||
/* Fancier color for constants such as None, True, and False. */
|
||||
.doc-signature .kc {
|
||||
color: var(--md-code-hl-constant-color);
|
||||
}
|
||||
|
||||
/* Fancier color for built-in types (only useful when cross-references are used). */
|
||||
.doc-signature .n > a[href^="https://docs.python.org/"][href*="/functions.html#"],
|
||||
.doc-signature .n > a[href^="https://docs.python.org/"][href*="/stdtypes.html#"] {
|
||||
color: var(--md-code-hl-constant-color);
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
# Development
|
||||
|
||||
This guide walks you through everything you need to develop, test, document, build, and release `pcdigitizer`.
|
||||
It assumes you're comfortable with a terminal but doesn't assume you've used any of these tools before.
|
||||
|
||||
## What is pcdigitizer?
|
||||
|
||||
`pcdigitizer` is a Python library that programmatically downloads, parses, and digitizes chemical data and annotations from [PubChem](https://pubchem.ncbi.nlm.nih.gov/).
|
||||
It talks to PubChem's REST API (called PUG-REST) and returns structured data using [Polars](https://pola.rs/) DataFrames.
|
||||
|
||||
The source code lives at [github.com/scienting/pcdigitizer](https://github.com/scienting/pcdigitizer).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before you begin, make sure you have:
|
||||
|
||||
- Git installed and configured with your GitHub credentials.
|
||||
If you're new to Git, GitHub's [Getting Started guide](https://docs.github.com/en/get-started) covers installation and setup.
|
||||
- A GitHub account with access to the `scienting/pcdigitizer` repository.
|
||||
- pixi; the project's environment and dependency manager (explained next).
|
||||
|
||||
You do *not* need to install Python yourself.
|
||||
Pixi handles that.
|
||||
|
||||
## Understanding pixi
|
||||
|
||||
`pcdigitizer` uses [pixi](https://pixi.sh/latest/) instead of the more common `pip` + `venv` workflow.
|
||||
Pixi is a package manager that creates isolated environments and installs both Python and all dependencies for you: conda packages and PyPI packages alike.
|
||||
This means every contributor works with the same Python version and the same library versions, which eliminates "works on my machine" problems.
|
||||
|
||||
### Installing pixi
|
||||
|
||||
Follow the instructions at [pixi.sh](https://pixi.sh/latest/) for your operating system.
|
||||
On macOS or Linux, the quickest method is:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://pixi.sh/install.sh | bash
|
||||
```
|
||||
|
||||
After installation, restart your terminal so the `pixi` command is available.
|
||||
Verify it worked:
|
||||
|
||||
```bash
|
||||
pixi --version
|
||||
```
|
||||
|
||||
### How pixi organizes this project
|
||||
|
||||
The file `pixi.toml` at the root of the repository defines everything pixi needs to know: which Python version to use, which libraries to install, and which shortcut commands (called *tasks*) are available.
|
||||
|
||||
`pcdigitizer` defines three environments:
|
||||
|
||||
| Environment | Purpose | How to activate |
|
||||
| :---------- | :------ | :-------------- |
|
||||
| `default` | The runtime dependencies only (what a user of the library needs). | `pixi shell` |
|
||||
| `dev` | Everything in default plus testing, linting, formatting, building, and publishing tools. | `pixi shell -e dev` |
|
||||
| `docs` | Everything needed to build and preview the documentation site. | `pixi shell -e docs` |
|
||||
|
||||
You'll use `dev` for most day-to-day work and `docs` when editing documentation.
|
||||
The separation keeps each environment lean: you don't need MkDocs installed to run tests, and you don't need pytest installed to preview docs.
|
||||
|
||||
## Setting up the Development Environment
|
||||
|
||||
1. Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone git@github.com:scienting/pcdigitizer.git
|
||||
cd pcdigitizer
|
||||
```
|
||||
|
||||
This creates a local copy of the codebase.
|
||||
The `git@github.com:` prefix uses SSH authentication.
|
||||
If you haven't set up SSH keys with GitHub, see [GitHub's SSH guide](https://docs.github.com/en/authentication/connecting-to-github-with-ssh).
|
||||
|
||||
2. Install dependencies:
|
||||
|
||||
```bash
|
||||
pixi install
|
||||
```
|
||||
|
||||
This reads `pixi.toml`, downloads Python 3.13+, and installs every dependency for all three environments.
|
||||
The first run takes a few minutes because it's fetching everything from scratch.
|
||||
Subsequent runs are fast because pixi caches packages.
|
||||
|
||||
3. Activate the development environment:
|
||||
|
||||
```bash
|
||||
pixi shell -e dev
|
||||
```
|
||||
|
||||
Your terminal prompt changes to indicate you're inside the pixi environment.
|
||||
Every command you run now (e.g., `python`, `pytest`, `ruff`) points to the versions pixi installed, not whatever might be on your system path.
|
||||
|
||||
When you're done working, type `exit` to leave the pixi shell.
|
||||
|
||||
Alternatively, you can run one-off commands without entering the shell by using `pixi run -e dev <command>`.
|
||||
|
||||
## Project Layout
|
||||
|
||||
Here's a high-level view of the repository.
|
||||
Source files inside `pcdigitizer/` aren't listed here because they change often; explore the package directory directly to see the current modules.
|
||||
|
||||
```text
|
||||
.
|
||||
├── pcdigitizer/ # The Python package (source code lives here)
|
||||
├── tests/ # Test suite (pytest)
|
||||
│ ├── conftest.py # Shared fixtures and auto-enables debug logging
|
||||
│ ├── test_*.py # Test modules
|
||||
│ └── tmp/ # Temporary test data
|
||||
├── docs/ # MkDocs documentation source
|
||||
│ ├── css/ # Custom stylesheets
|
||||
│ ├── js/ # JavaScript (MathJax config, etc.)
|
||||
│ ├── img/ # Images used in docs
|
||||
│ ├── gen_ref_pages.py # Script that auto-generates API reference pages
|
||||
│ ├── pages.yml # Navigation structure
|
||||
│ └── index.md # Documentation home page
|
||||
├── pixi.toml # Pixi configuration (environments, dependencies, tasks)
|
||||
├── pixi.lock # Locked dependency versions (committed to Git)
|
||||
├── pyproject.toml # Python packaging metadata
|
||||
├── mkdocs.yml # MkDocs site configuration
|
||||
├── CHANGELOG.md # Release history
|
||||
├── LICENSE.md # Prosperity license
|
||||
└── README.md
|
||||
```
|
||||
|
||||
A few things worth noting about this layout:
|
||||
|
||||
The `pixi.lock` file pins every dependency to an exact version.
|
||||
It's committed to Git so that `pixi install` reproduces the same environment on every machine.
|
||||
You should never edit it by hand; pixi updates it automatically when you change `pixi.toml`.
|
||||
|
||||
The `pyproject.toml` file contains Python packaging metadata (package name, author, build system).
|
||||
It works alongside `pixi.toml`, which handles the environment and task definitions.
|
||||
|
||||
The package installs in *editable mode* (note the `editable = true` in `pixi.toml`), which means changes you make to the source files take effect immediately without reinstalling.
|
||||
|
||||
## Code Formatting and Linting
|
||||
|
||||
`pcdigitizer` uses [Ruff](https://docs.astral.sh/ruff/) for both linting (catching potential bugs and style violations) and formatting (enforcing consistent code style).
|
||||
Ruff is extremely fast, it replaces tools like flake8, isort, and black in a single binary.
|
||||
|
||||
To lint and format the entire codebase:
|
||||
|
||||
```bash
|
||||
pixi run format
|
||||
```
|
||||
|
||||
This command does two things in sequence (the `format` task depends on the `lint` task):
|
||||
|
||||
1. Lints the code with `ruff check --fix`, which catches errors and auto-fixes what it can.
|
||||
2. Sorts imports and formats the code with `ruff format`, which enforces consistent spacing, line length, quote style, and so on.
|
||||
|
||||
Run this before every commit.
|
||||
If Ruff finds issues it can't auto-fix, it prints them to the terminal with file names and line numbers.
|
||||
|
||||
You can also lint without formatting:
|
||||
|
||||
```bash
|
||||
pixi run lint
|
||||
```
|
||||
|
||||
The rules Ruff enforces are defined in `.ruff.toml` at the repository root.
|
||||
|
||||
## Documentation
|
||||
|
||||
The documentation site is built with [MkDocs](https://www.mkdocs.org/) using the [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) theme.
|
||||
The site configuration lives in `mkdocs.yml`, and the source files live in the `docs/` directory.
|
||||
|
||||
A few things worth knowing about the documentation setup:
|
||||
|
||||
- API reference pages are generated automatically.
|
||||
The `mkdocstrings` plugin reads docstrings from the Python source code and renders them as formatted documentation.
|
||||
If you write or update a docstring in the code, the API docs update too.
|
||||
- Docstrings follow Google style.
|
||||
When writing docstrings, use the [Google docstring format](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings).
|
||||
- Math is supported.
|
||||
The site loads MathJax, so you can write LaTeX in your documentation files.
|
||||
|
||||
### Previewing documentation locally
|
||||
|
||||
```bash
|
||||
pixi run -e docs docs-serve
|
||||
```
|
||||
|
||||
This starts a local web server.
|
||||
Open [http://127.0.0.1:8000/](http://127.0.0.1:8000/) in your browser.
|
||||
The page reloads automatically whenever you save a file, so you can edit and preview side by side.
|
||||
|
||||
Press `Ctrl+C` in the terminal to stop the server.
|
||||
|
||||
### Building documentation for deployment
|
||||
|
||||
```bash
|
||||
pixi run -e docs docs
|
||||
```
|
||||
|
||||
This generates the static site in a `public/` directory (deleting any previous build first).
|
||||
The CI/CD pipeline typically runs this command to produce the files that get deployed.
|
||||
|
||||
## Testing
|
||||
|
||||
Tests live in the `tests/` directory and run with [pytest](https://docs.pytest.org/).
|
||||
The test configuration is in `.pytest.ini`.
|
||||
|
||||
A shared fixture in `conftest.py` automatically turns on debug-level logging for every test session, so you can see detailed log output when something goes wrong.
|
||||
|
||||
### Running tests
|
||||
|
||||
```bash
|
||||
pixi run -e dev tests
|
||||
```
|
||||
|
||||
Under the hood, this runs pytest with coverage tracking enabled.
|
||||
It produces two reports: an XML coverage report (`report.xml`) and a JUnit-style test report, both used by CI pipelines.
|
||||
The `--failed-first` flag re-runs previously failing tests first, which speeds up debugging.
|
||||
|
||||
### Checking test coverage
|
||||
|
||||
After running the tests, you can see a summary of which lines are covered:
|
||||
|
||||
```bash
|
||||
pixi run -e dev coverage
|
||||
```
|
||||
|
||||
This prints a table showing each source file and its coverage percentage.
|
||||
Look for files with low coverage, those are good candidates for new tests.
|
||||
|
||||
### Writing tests
|
||||
|
||||
Place new test files in the `tests/` directory.
|
||||
Pytest discovers any file named `test_*.py` or `*_test.py` automatically.
|
||||
A minimal test looks like:
|
||||
|
||||
```python
|
||||
from pcdigitizer import PubChemAPI
|
||||
|
||||
|
||||
def test_build_url_basic():
|
||||
url = PubChemAPI.build_url(
|
||||
domain="compound",
|
||||
namespace="name",
|
||||
identifiers="aspirin",
|
||||
)
|
||||
assert "compound" in url
|
||||
assert "aspirin" in url
|
||||
```
|
||||
|
||||
Keep in mind that tests calling PubChem's live API will make real HTTP requests.
|
||||
For unit tests, consider mocking the network layer with `unittest.mock.patch` or a library like `responses`.
|
||||
|
||||
## Logging
|
||||
|
||||
`pcdigitizer` uses [Loguru](https://loguru.readthedocs.io/) for logging.
|
||||
By default, logging is completely disabled; the library stays silent so it doesn't clutter output for end users.
|
||||
When you need to see what the library is doing (debugging a failed API call, tracing how data gets parsed, confirming a request URL), you turn logging on explicitly.
|
||||
|
||||
### Log levels
|
||||
|
||||
Python's logging system uses numeric levels to control how much detail you see.
|
||||
Each level includes everything above it: setting the level to `WARNING` shows warnings and errors but hides informational and debug messages.
|
||||
Setting it to `DEBUG` shows everything.
|
||||
|
||||
| Level | Number | What it captures | When to use it |
|
||||
| :---- | :----- | :--------------- | :------------- |
|
||||
| `DEBUG` | 10 | Every internal detail: constructed URLs, raw response status codes, intermediate parsing steps. | Troubleshooting a specific bug. You want to see exactly what `pcdigitizer` sends to PubChem and what it gets back. This is the level the test suite uses. |
|
||||
| `INFO` | 20 | High-level progress: which API endpoint was called, how many records were returned, which data source was queried. | Day-to-day development or monitoring a long-running script. Enough to confirm things are working without flooding the terminal. |
|
||||
| `WARNING` | 30 | Unexpected but non-fatal situations: a deprecated endpoint, a retry after a transient failure, a missing optional field in the response. | Production scripts where you only want to hear about potential problems. |
|
||||
| `ERROR` | 40 | Something went wrong and the operation couldn't complete: a failed HTTP request, an unparseable response, an invalid identifier. | Same as `WARNING`, but stricter. You only see actual failures. |
|
||||
| `CRITICAL` | 50 | A catastrophic problem that likely means the program can't continue. | Rarely relevant for a library like `pcdigitizer`, but available if needed. |
|
||||
|
||||
If you're unsure which level to pick, start with `INFO` (20).
|
||||
Drop to `DEBUG` (10) when you need to dig into a specific problem.
|
||||
|
||||
### Enabling logging in code
|
||||
|
||||
Call `enable_logging` at the top of your script, before you make any API calls:
|
||||
|
||||
```python
|
||||
from pcdigitizer import enable_logging
|
||||
|
||||
enable_logging(level_set=20) # INFO: shows progress without excessive detail
|
||||
```
|
||||
|
||||
The `enable_logging` function accepts several arguments:
|
||||
|
||||
```python
|
||||
enable_logging(
|
||||
level_set=10, # Log level (10=DEBUG, 20=INFO, etc.)
|
||||
stdout_set=True, # Print logs to the terminal
|
||||
file_path="/tmp/pcdigitizer.log", # Also write logs to this file (optional)
|
||||
)
|
||||
```
|
||||
|
||||
When `stdout_set` is `True` (the default), log messages print to the terminal with color-coded formatting: timestamps in green, the log level highlighted by severity, and the source file, function, and line number in cyan.
|
||||
When you provide a `file_path`, the same messages are also written to that file so you can review them later.
|
||||
|
||||
### Enabling logging with environment variables
|
||||
|
||||
If you don't want to modify code, say you're running someone else's script and need to see what's happening, you can enable logging through environment variables:
|
||||
|
||||
```bash
|
||||
export PCDIGITIZER_LOG=True
|
||||
export PCDIGITIZER_LOG_LEVEL=10
|
||||
```
|
||||
|
||||
`pcdigitizer` reads these variables at import time
|
||||
If `PCDIGITIZER_LOG` is `True`, it calls `enable_logging` automatically with whatever level, output, and file path you've specified.
|
||||
The full set of variables:
|
||||
|
||||
| Variable | Default | What it controls |
|
||||
| :------- | :------ | :--------------- |
|
||||
| `PCDIGITIZER_LOG` | `False` | Whether logging is enabled at all. Must be `True` to activate. |
|
||||
| `PCDIGITIZER_LOG_LEVEL` | `20` | The numeric log level (see the table above). |
|
||||
| `PCDIGITIZER_STDOUT` | `True` | Whether to print logs to the terminal. |
|
||||
| `PCDIGITIZER_LOG_FILE_PATH` | None | Path to a log file. Omit this to skip file logging. |
|
||||
|
||||
### Logging in tests
|
||||
|
||||
The test suite enables debug-level logging automatically.
|
||||
The fixture in `conftest.py` calls `enable_logging(10)` at the start of every test session, so when a test fails you'll see the full trace of API calls and internal operations in the pytest output without any extra setup.
|
||||
|
||||
## Building the Package
|
||||
|
||||
When you're ready to create a distributable package (a `.tar.gz` and a `.whl` file):
|
||||
|
||||
```bash
|
||||
pixi run build
|
||||
```
|
||||
|
||||
This first removes any previous `build/` directory (to avoid stale artifacts), then runs `python3 -m build` to produce fresh distribution files in the `dist/` directory.
|
||||
|
||||
The build uses `setuptools` and `setuptools-scm`.
|
||||
The `setuptools-scm` plugin derives the package version from Git tags automatically, so the version in `pixi.toml` stays in sync with the version embedded in the built package.
|
||||
|
||||
## Versioning and Releases
|
||||
|
||||
`pcdigitizer` uses [bump-my-version](https://github.com/callowayproject/bump-my-version) to manage version numbers.
|
||||
The current version is stored in `pixi.toml` (currently `26.4.10`).
|
||||
|
||||
### Bumping the version
|
||||
|
||||
```bash
|
||||
pixi run -e dev bump
|
||||
```
|
||||
|
||||
This increments the patch number (e.g., `26.4.10` → `26.4.11`), updates the version string in the configured files, and creates a Git commit and tag.
|
||||
For minor or major bumps, you'd run `bump-my-version bump minor` or `bump-my-version bump major` directly inside the dev shell.
|
||||
|
||||
### Publishing to PyPI
|
||||
|
||||
Publishing is a two-step process: build, then upload.
|
||||
|
||||
1. Build the package (if you haven't already):
|
||||
|
||||
```bash
|
||||
pixi run build
|
||||
```
|
||||
|
||||
2. Publish to TestPyPI first to verify everything works:
|
||||
|
||||
```bash
|
||||
pixi run publish-test
|
||||
```
|
||||
|
||||
TestPyPI is a separate package index that mirrors PyPI's infrastructure.
|
||||
Publishing there lets you verify the package installs correctly without affecting real users.
|
||||
You can install from TestPyPI with:
|
||||
|
||||
```bash
|
||||
pip install --index-url https://test.pypi.org/simple/ pcdigitizer
|
||||
```
|
||||
|
||||
3. Publish to production PyPI once you're confident:
|
||||
|
||||
```bash
|
||||
pixi run publish
|
||||
```
|
||||
|
||||
Both commands use [twine](https://twine.readthedocs.io/) under the hood.
|
||||
You'll need PyPI credentials configured either through a `~/.pypirc` file or environment variables.
|
||||
See [twine's documentation](https://twine.readthedocs.io/en/stable/#configuration) for setup instructions.
|
||||
|
||||
## Quick Reference: All pixi Tasks
|
||||
|
||||
| Task | Environment | What it does |
|
||||
| :--- | :---------: | :----------- |
|
||||
| `pixi run lint` | dev | Lint the codebase with Ruff and auto-fix where possible |
|
||||
| `pixi run format` | dev | Lint, sort imports, and format all code |
|
||||
| `pixi run -e dev tests` | dev | Run the test suite with coverage tracking |
|
||||
| `pixi run -e dev coverage` | dev | Print a coverage summary table |
|
||||
| `pixi run -e dev bump` | dev | Increment the patch version number |
|
||||
| `pixi run build` | dev | Clean and build distribution files |
|
||||
| `pixi run publish-test` | dev | Upload to TestPyPI |
|
||||
| `pixi run publish` | dev | Upload to production PyPI |
|
||||
| `pixi run -e docs docs-serve` | docs | Serve documentation locally with live reload |
|
||||
| `pixi run -e docs docs` | docs | Build the documentation site to `public/` |
|
||||
|
||||
## Maintenance Best Practices
|
||||
|
||||
Keep your local copy current by pulling from `main` regularly:
|
||||
|
||||
```bash
|
||||
git pull origin main
|
||||
```
|
||||
|
||||
After pulling, run `pixi install` to pick up any new or changed dependencies.
|
||||
|
||||
Review open issues and pull requests on GitHub frequently.
|
||||
Write clear commit messages that explain *why* a change was made, not just *what* changed.
|
||||
When updating dependencies, run the full test suite before pushing to make sure nothing broke.
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Generate the code reference pages."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import mkdocs_gen_files
|
||||
|
||||
SRC_DIR = "pcdigitizer"
|
||||
WRITE_DIR = "api"
|
||||
|
||||
for path in sorted(Path(SRC_DIR).rglob("*.py")): #
|
||||
module_path = path.relative_to(SRC_DIR).with_suffix("") #
|
||||
|
||||
doc_path = path.relative_to(SRC_DIR).with_suffix(".md") #
|
||||
|
||||
if not os.path.exists(Path(WRITE_DIR)):
|
||||
os.mkdir(Path(WRITE_DIR))
|
||||
|
||||
full_doc_path = Path(WRITE_DIR, doc_path) #
|
||||
|
||||
parts = list(module_path.parts)
|
||||
|
||||
if parts[-1] == "__init__": #
|
||||
parts = parts[:-1]
|
||||
elif parts[-1] == "__main__":
|
||||
continue
|
||||
|
||||
if len(parts) == 0:
|
||||
continue
|
||||
|
||||
with mkdocs_gen_files.open(full_doc_path, "w") as fd: #
|
||||
identifier = ".".join(parts) #
|
||||
|
||||
print("::: " + identifier, file=fd) #
|
||||
|
||||
mkdocs_gen_files.set_edit_path(full_doc_path, path) #
|
||||
@@ -0,0 +1,64 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
viewBox="0 0 14.602636 14.356544"
|
||||
preserveAspectRatio="xMidYMid meet"
|
||||
focusable="false"
|
||||
style="display:block"
|
||||
width="157.28125mm"
|
||||
height="95.27916mm"
|
||||
version="1.1"
|
||||
id="svg14"
|
||||
sodipodi:docname="colab.svg"
|
||||
inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:svg="http://www.w3.org/2000/svg">
|
||||
<defs
|
||||
id="defs18" />
|
||||
<sodipodi:namedview
|
||||
id="namedview16"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pagecheckerboard="0"
|
||||
showgrid="false"
|
||||
inkscape:zoom="0.84453451"
|
||||
inkscape:cx="143.27419"
|
||||
inkscape:cy="431.00666"
|
||||
inkscape:window-width="1868"
|
||||
inkscape:window-height="1016"
|
||||
inkscape:window-x="1972"
|
||||
inkscape:window-y="27"
|
||||
inkscape:window-maximized="1"
|
||||
inkscape:current-layer="svg14"
|
||||
units="mm"
|
||||
height="95.27916mm"
|
||||
inkscape:document-units="mm" />
|
||||
<g
|
||||
id="g12"
|
||||
transform="translate(-4.7003952,-4.8209419)">
|
||||
<path
|
||||
d="M 4.54,9.46 2.19,7.1 a 6.93,6.93 0 0 0 0,9.79 L 4.55,14.53 A 3.59,3.59 0 0 1 4.54,9.46 Z"
|
||||
fill="#e8710a"
|
||||
id="path2" />
|
||||
<path
|
||||
d="m 2.19,7.1 2.35,2.36 a 3.59,3.59 0 0 1 5.08,0 l 1.71,-2.93 v 0 l -0.1,-0.08 v 0 A 6.93,6.93 0 0 0 2.19,7.1 Z"
|
||||
fill="#f9ab00"
|
||||
id="path4" />
|
||||
<path
|
||||
d="m 11.34,17.46 v 0 L 9.62,14.54 a 3.59,3.59 0 0 1 -5.08,0 L 2.19,16.9 a 6.93,6.93 0 0 0 9,0.65 l 0.11,-0.09"
|
||||
fill="#f9ab00"
|
||||
id="path6" />
|
||||
<path
|
||||
d="m 12,7.1 a 6.93,6.93 0 0 0 0,9.79 l 2.36,-2.36 A 3.5921024,3.5921024 0 1 1 19.44,9.45 L 21.81,7.1 A 6.93,6.93 0 0 0 12,7.1 Z"
|
||||
fill="#f9ab00"
|
||||
id="path8" />
|
||||
<path
|
||||
d="m 21.81,7.1 -2.35,2.36 a 3.5921024,3.5921024 0 0 1 -5.08,5.08 L 12,16.9 a 6.9331829,6.9331829 0 0 0 9.81,-9.8 z"
|
||||
fill="#e8710a"
|
||||
id="path10" />
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.1 KiB |
@@ -0,0 +1,2 @@
|
||||
--8<-- "README.md"
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
window.MathJax = {
|
||||
tex: {
|
||||
inlineMath: [["\\(", "\\)"], ["$", "$"]],
|
||||
displayMath: [["\\[", "\\]"], ["$$", "$$"]],
|
||||
processEscapes: true,
|
||||
processEnvironments: true
|
||||
},
|
||||
options: {
|
||||
ignoreHtmlClass: ".*|",
|
||||
processHtmlClass: "arithmatex"
|
||||
}
|
||||
};
|
||||
|
||||
document$.subscribe(() => {
|
||||
MathJax.startup.output.clearCache()
|
||||
MathJax.typesetClear()
|
||||
MathJax.texReset()
|
||||
MathJax.typesetPromise()
|
||||
})
|
||||
@@ -0,0 +1,11 @@
|
||||
nav:
|
||||
- Home: index.md
|
||||
- API: api
|
||||
- Development: development.md
|
||||
|
||||
sort:
|
||||
type: natural
|
||||
ignore_case: true
|
||||
by: title
|
||||
direction: asc
|
||||
flatten_single_child_sections: true
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
docs_dir: docs
|
||||
|
||||
site_name: pcdigitizer
|
||||
site_author: Scientific Computing Studio
|
||||
|
||||
repo_name: scienting/pcdigitizer
|
||||
repo_url: https://github.com/scienting/pcdigitizer
|
||||
copyright: <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/">CC BY-NC-SA 4.0</a> by Scientific Computing Studio
|
||||
|
||||
# https://squidfunk.github.io/mkdocs-material/
|
||||
theme:
|
||||
name: material
|
||||
custom_dir: docs/.overrides
|
||||
language: en
|
||||
palette:
|
||||
# Palette toggle for light mode
|
||||
- scheme: default
|
||||
toggle:
|
||||
icon: material/lightbulb-outline
|
||||
name: Switch to dark mode
|
||||
|
||||
# Palette toggle for dark mode
|
||||
- scheme: dark
|
||||
toggle:
|
||||
icon: material/lightbulb
|
||||
name: Switch to light mode
|
||||
font:
|
||||
text: Roboto
|
||||
code: Roboto Mono
|
||||
icon:
|
||||
repo: fontawesome/brands/github
|
||||
annotation: material/star-four-points-circle
|
||||
features:
|
||||
- content.code.annotate
|
||||
- content.code.copy
|
||||
- content.code.select
|
||||
- navigation.tabs
|
||||
- navigation.tabs.sticky
|
||||
- navigation.tracking
|
||||
- navigation.top
|
||||
- navigation.indexes
|
||||
- navigation.path
|
||||
- navigation.prune
|
||||
- toc.follow
|
||||
- search.suggest
|
||||
|
||||
validation:
|
||||
omitted_files: warn
|
||||
absolute_links: warn
|
||||
unrecognized_links: warn
|
||||
|
||||
# Options need to be indented twice for some reason?
|
||||
plugins:
|
||||
- search
|
||||
- autorefs
|
||||
- gen-files:
|
||||
scripts:
|
||||
- docs/gen_ref_pages.py
|
||||
- mkdocstrings:
|
||||
handlers:
|
||||
python:
|
||||
inventories:
|
||||
- "https://docs.python.org/3/objects.inv"
|
||||
- "https://requests.readthedocs.io/en/latest/objects.inv"
|
||||
paths: ["pcdigitizer"]
|
||||
options:
|
||||
show_source: false
|
||||
show_root_heading: false
|
||||
annotations_path: brief
|
||||
docstring_style: google
|
||||
merge_init_into_class: true
|
||||
docstring_section_style: spacy
|
||||
show_if_no_docstring: true
|
||||
show_labels: false
|
||||
parameter_headings: true
|
||||
show_symbol_type_heading: true
|
||||
show_symbol_type_toc: true
|
||||
filters:
|
||||
- ""
|
||||
- awesome-nav
|
||||
- git-revision-date-localized:
|
||||
type: iso_datetime
|
||||
timezone: America/Detroit
|
||||
fallback_to_build_date: true
|
||||
- macros
|
||||
- glightbox
|
||||
|
||||
extra:
|
||||
generator: false
|
||||
|
||||
extra_css:
|
||||
- css/base.css
|
||||
- css/colors.css
|
||||
- css/mkdocstrings.css
|
||||
|
||||
extra_javascript:
|
||||
- https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js
|
||||
- js/mathjax-config.js
|
||||
|
||||
markdown_extensions:
|
||||
- abbr
|
||||
- toc:
|
||||
permalink: true
|
||||
- admonition
|
||||
- attr_list
|
||||
- def_list
|
||||
- footnotes
|
||||
- md_in_html
|
||||
- tables
|
||||
- pymdownx.arithmatex:
|
||||
generic: true
|
||||
- pymdownx.betterem
|
||||
- pymdownx.caret
|
||||
- pymdownx.details
|
||||
- pymdownx.highlight:
|
||||
anchor_linenums: true
|
||||
line_spans: __span
|
||||
pygments_lang_class: true
|
||||
- pymdownx.inlinehilite
|
||||
- pymdownx.keys
|
||||
- pymdownx.mark
|
||||
- pymdownx.smartsymbols
|
||||
- pymdownx.snippets
|
||||
- pymdownx.superfences:
|
||||
custom_fences:
|
||||
- name: mermaid
|
||||
class: mermaid
|
||||
format: !!python/name:pymdownx.superfences.fence_code_format
|
||||
- pymdownx.tabbed:
|
||||
alternate_style: true
|
||||
- pymdownx.tasklist:
|
||||
custom_checkbox: true
|
||||
- pymdownx.tilde
|
||||
- pymdownx.emoji:
|
||||
emoji_index: !!python/name:material.extensions.emoji.twemoji
|
||||
emoji_generator: !!python/name:material.extensions.emoji.to_svg
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Programmatically download, parse, and digitize chemical data and annotations from PubChem"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from ast import literal_eval
|
||||
from typing import Any
|
||||
|
||||
from . import responses
|
||||
from .data import Annotation
|
||||
from .pubchem import PubChemAPI
|
||||
from .task import GetAnnotationPage
|
||||
|
||||
__all__ = ["responses", "Annotation", "PubChemAPI", "GetAnnotationPage"]
|
||||
|
||||
from loguru import logger
|
||||
|
||||
logger.disable("pcdigitizer")
|
||||
|
||||
LOG_FORMAT = (
|
||||
"<green>{time:HH:mm:ss}</green> | "
|
||||
"<level>{level: <8}</level> | "
|
||||
"<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
|
||||
)
|
||||
|
||||
|
||||
def enable_logging(
|
||||
level_set: int,
|
||||
stdout_set: bool = True,
|
||||
file_path: str | None = None,
|
||||
log_format: str = LOG_FORMAT,
|
||||
colorize: bool = True,
|
||||
) -> None:
|
||||
r"""Enable logging.
|
||||
|
||||
Args:
|
||||
level: Requested log level: `10` is debug, `20` is info.
|
||||
file_path: Also write logs to files here.
|
||||
"""
|
||||
config: dict[str, Any] = {"handlers": []}
|
||||
if stdout_set:
|
||||
config["handlers"].append(
|
||||
{
|
||||
"sink": sys.stdout,
|
||||
"level": level_set,
|
||||
"format": log_format,
|
||||
"colorize": colorize,
|
||||
}
|
||||
)
|
||||
if isinstance(file_path, str):
|
||||
config["handlers"].append(
|
||||
{
|
||||
"sink": file_path,
|
||||
"level": level_set,
|
||||
"format": log_format,
|
||||
"colorize": colorize,
|
||||
}
|
||||
)
|
||||
# https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.configure
|
||||
logger.configure(**config)
|
||||
|
||||
logger.enable("pcdigitizer")
|
||||
|
||||
|
||||
if literal_eval(os.environ.get("PCDIGITIZER_LOG", "False")):
|
||||
level = int(os.environ.get("PCDIGITIZER_LOG_LEVEL", 20))
|
||||
stdout = literal_eval(os.environ.get("PCDIGITIZER_STDOUT", "True"))
|
||||
log_file_path = os.environ.get("PCDIGITIZER_LOG_FILE_PATH", None)
|
||||
enable_logging(level, stdout, log_file_path)
|
||||
@@ -0,0 +1,11 @@
|
||||
from .annotations import Annotation, AnnotationProcessor
|
||||
from .dissociation_constant import DissociationConstantData
|
||||
from .registry import ANNOTATION_REGISTRY, get_processor
|
||||
|
||||
__all__ = [
|
||||
"Annotation"
|
||||
"AnnotationProcessor",
|
||||
"DissociationConstantData",
|
||||
"ANNOTATION_REGISTRY",
|
||||
"get_processor",
|
||||
]
|
||||
@@ -0,0 +1,32 @@
|
||||
from enum import StrEnum
|
||||
from typing import Protocol
|
||||
|
||||
import polars as pl
|
||||
|
||||
from pcdigitizer.responses import AnnotationEntry
|
||||
|
||||
|
||||
class Annotation(StrEnum):
|
||||
"""PubChem annotation headings supported by this package.
|
||||
|
||||
Each member corresponds to a heading in the PubChem PUG-View
|
||||
`/annotations/heading/<heading>` endpoint and maps to a registered
|
||||
processor class that knows how to parse the raw API response for that
|
||||
heading into a tidy polars DataFrame.
|
||||
|
||||
Because `Annotation` extends `StrEnum`, members can be passed
|
||||
directly to any function expecting a `str` (e.g.
|
||||
`PubChemAPI.get_data`) without accessing `.value`.
|
||||
|
||||
To add support for a new heading, add a member here and register its
|
||||
processor in the heading registry.
|
||||
"""
|
||||
|
||||
DISSOCIATION_CONSTANTS = "Dissociation Constants"
|
||||
|
||||
|
||||
class AnnotationProcessor(Protocol):
|
||||
"""Interface that any annotation-specific data processor must satisfy."""
|
||||
|
||||
@classmethod
|
||||
def from_page(cls, annotation_data: list[AnnotationEntry]) -> pl.DataFrame: ...
|
||||
@@ -0,0 +1,520 @@
|
||||
"""Parsing and extraction utilities for PubChem dissociation constant data.
|
||||
|
||||
This module processes raw annotation entries from the PubChem PUG-View API
|
||||
for the "Dissociation Constants" heading. Each entry contains one or more
|
||||
free-text pKa strings deposited by data providers, which are highly
|
||||
inconsistent in format. The parsing pipeline normalizes these strings into
|
||||
structured records suitable for downstream analysis.
|
||||
|
||||
## Typical usage
|
||||
|
||||
Fetch data via [`PubChemAPI`][pcdigitizer.pubchem.PubChemAPI] and pass it
|
||||
directly to
|
||||
[`DissociationConstantData.from_page`][pcdigitizer.data.dissociation_constant.DissociationConstantData.from_page]
|
||||
|
||||
```python
|
||||
from pcdigitizer import PubChemAPI, Annotation
|
||||
from pcdigitizer.data import DissociationConstantData
|
||||
|
||||
raw = PubChemAPI.get_data(Annotation.DISSOCIATION_CONSTANTS)
|
||||
df = DissociationConstantData.from_page(raw)
|
||||
```
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Iterable, TypedDict
|
||||
|
||||
import polars as pl
|
||||
from loguru import logger
|
||||
|
||||
from pcdigitizer.data import AnnotationProcessor
|
||||
from pcdigitizer.responses import AnnotationEntry, PubChemDatum
|
||||
|
||||
|
||||
class ParsedPKa(TypedDict):
|
||||
"""A single parsed pKa value extracted from a free-text string."""
|
||||
|
||||
pka_label: str | None
|
||||
"""The label identifying which ionization site this value belongs to
|
||||
(e.g. `"pKa1"`, `"pK2"`). `None` when the source text does not include a label.
|
||||
"""
|
||||
|
||||
pka_value: float
|
||||
"""The numeric pKa value."""
|
||||
|
||||
temperature_C: float | None
|
||||
"""
|
||||
The temperature in degrees Celsius at which the measurement was made, if stated.
|
||||
`None` when not specified.
|
||||
"""
|
||||
|
||||
comment: str | None
|
||||
"""
|
||||
The original source line with commas replaced by semicolons,
|
||||
retained for provenance. `None` for multi-value sentence parses.
|
||||
"""
|
||||
|
||||
|
||||
class FlatPKaRecord(TypedDict):
|
||||
"""
|
||||
A fully resolved pKa record with compound and source identifiers.
|
||||
"""
|
||||
|
||||
cid: int
|
||||
"""PubChem Compound Identifier."""
|
||||
|
||||
sid: int
|
||||
"""PubChem Substance Identifier for the depositing source record."""
|
||||
|
||||
pclid: int | None
|
||||
"""
|
||||
PubChem Live Data Identifier linking to the specific measurement record,
|
||||
if available.
|
||||
"""
|
||||
|
||||
pka_label: str | None
|
||||
"""See [`ParsedPKa`][data.dissociation_constant.ParsedPKa.pka_label]."""
|
||||
|
||||
pka_value: float
|
||||
"""See [`ParsedPKa`][data.dissociation_constant.ParsedPKa.pka_value]."""
|
||||
|
||||
temperature_C: float | None
|
||||
"""See [`ParsedPKa`][data.dissociation_constant.ParsedPKa.temperature_C]."""
|
||||
|
||||
comment: str | None
|
||||
"""See [`ParsedPKa`][data.dissociation_constant.ParsedPKa.comment]."""
|
||||
|
||||
|
||||
_OUTPUT_SCHEMA: dict[str, type[pl.DataType]] = {
|
||||
"cid": pl.Int64,
|
||||
"sid": pl.Int64,
|
||||
"pclid": pl.Int64,
|
||||
"pka_label": pl.String,
|
||||
"pka_value": pl.Float64,
|
||||
"temperature_C": pl.Float64,
|
||||
"comment": pl.String,
|
||||
}
|
||||
"""Expected output schema (used to return an empty DataFrame safely)"""
|
||||
|
||||
|
||||
class DissociationConstantData(AnnotationProcessor):
|
||||
"""Parse and assemble PubChem dissociation constant annotation data.
|
||||
|
||||
All methods are static or class methods. The primary public interface is
|
||||
[`from_page`][data.dissociation_constant.DissociationConstantData.from_page],
|
||||
which converts a raw list of PubChem annotation
|
||||
entries into a tidy polars DataFrame.
|
||||
|
||||
The parsing pipeline for free-text pKa strings is the following.
|
||||
|
||||
1. [`parse_value`][data.dissociation_constant.DissociationConstantData.parse_value]
|
||||
is the top-level dispatcher. It first checks for
|
||||
the "pKa values are X, Y, and Z" sentence form via
|
||||
[`_parse_multi_value_sentence`][data.dissociation_constant.DissociationConstantData._parse_multi_value_sentence].
|
||||
If that does not match it splits the input on semicolons and delegates each
|
||||
segment to
|
||||
[`_parse_part`][data.dissociation_constant.DissociationConstantData._parse_part].
|
||||
|
||||
2. [`_parse_part`][data.dissociation_constant.DissociationConstantData._parse_part]
|
||||
tries each compiled pattern in
|
||||
[`_PATTERNS`][data.dissociation_constant.DissociationConstantData._PATTERNS]
|
||||
in priority order via
|
||||
[`_try_patterns`][data.dissociation_constant.DissociationConstantData._try_patterns],
|
||||
returning the first successful
|
||||
[`ParsedPKa`][data.dissociation_constant.ParsedPKa] or `None`.
|
||||
|
||||
3. Patterns are compiled once at class definition time and reused across all calls.
|
||||
|
||||
The following free-text forms are recognized (case-insensitive):
|
||||
|
||||
- `pKa values are 3.25, 4.76, and 6.17` (multi-value sentence)
|
||||
- `pKa3 = -2.03`
|
||||
- `pK1 = 2.36 (SRC: carboxylic acid)`
|
||||
- `pKa = 10.4 at 40 °C (tertiary amine)`
|
||||
- `Weak acid. pK (25 °C): 3.35`
|
||||
- `pKa = 0.7 (caffeine cation)`
|
||||
- `pKa = 20`
|
||||
|
||||
Lines that cannot be matched (e.g. density or solubility values
|
||||
mistakenly deposited under this heading) are logged at WARNING level
|
||||
and excluded from the output.
|
||||
"""
|
||||
|
||||
_PATTERN_LABELED: re.Pattern[str] = re.compile(
|
||||
r"""
|
||||
^\s*
|
||||
(?P<label>pK(?:a)?\d*) # pK, pKa, pK1, pKa2, etc.
|
||||
\s*(?:=|:)\s*
|
||||
(?P<value>-?[\d.]+) # numeric value, allow negative
|
||||
(?:\s*\(SRC:\s*(?P<comment>[^)]+)\))? # optional (SRC: ...)
|
||||
(?:\s+at\s+(?P<temp>[\d.]+)\s*°?C)? # optional "at XX °C"
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
"""
|
||||
Labeled pKa with optional SRC comment and optional temperature.
|
||||
Matches:
|
||||
|
||||
- `pKa3 = -2.03, pK1 = 2.36 (SRC: carboxylic acid), pKa = 10.4 at 40 °C`
|
||||
"""
|
||||
|
||||
_PATTERN_TEMP_PREFIX: re.Pattern[str] = re.compile(
|
||||
r"""
|
||||
(?P<label>pK(?:a)?) # pK or pKa
|
||||
\s*
|
||||
(?:\(\s*(?P<temp>[\d.]+)\s*°?C\))? # optional (25 °C)
|
||||
\s*[:=]\s*
|
||||
(?P<value>-?[\d.]+) # numeric value
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
"""
|
||||
temperature-prefix form.
|
||||
Matches:
|
||||
|
||||
- `Weak acid. pK (25 °C): 3.35, pK (25 °C) = 4.5`
|
||||
"""
|
||||
|
||||
# Pattern 3: fallback for bare numeric values in an environment context.
|
||||
# Matches: 4.24 in water, -1.34
|
||||
# Only used when a label group is present from one of the above patterns;
|
||||
# this pattern has no label group intentionally, so bare numbers without
|
||||
# any pK context are always rejected (see _parse_part for the label guard).
|
||||
_PATTERN_FALLBACK: re.Pattern[str] = re.compile(
|
||||
r"""
|
||||
^\s*
|
||||
(?P<value>-?[\d.]+)
|
||||
(?:\s+in\s+(?P<env>.+))? # optional "in water / solvent" context
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
"""
|
||||
Fallback for bare numeric values in an environment context.
|
||||
Matches:
|
||||
|
||||
- `4.24 in water, -1.34`
|
||||
|
||||
Only used when a label group is present from one of the above patterns;
|
||||
this pattern has no label group intentionally, so bare numbers without
|
||||
any pK context are always rejected (see
|
||||
[`_parse_part`][data.dissociation_constant.DissociationConstantData._parse_part]
|
||||
for the label guard).
|
||||
"""
|
||||
|
||||
_PATTERNS: list[re.Pattern[str]] = [
|
||||
_PATTERN_LABELED,
|
||||
_PATTERN_TEMP_PREFIX,
|
||||
_PATTERN_FALLBACK,
|
||||
]
|
||||
"""Ordered list used by _try_patterns; higher-specificity patterns first."""
|
||||
|
||||
_PATTERN_MULTI_VALUE: re.Pattern[str] = re.compile(
|
||||
r"pKa values are\s+([\d\.,\sand-]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
"""Pre-compiled pattern for the multi-value sentence form."""
|
||||
|
||||
@classmethod
|
||||
def _parse_multi_value_sentence(cls, line: str) -> list[ParsedPKa] | None:
|
||||
"""Attempt to parse the "pKa values are X, Y, and Z" sentence form.
|
||||
|
||||
This form lists multiple unlabeled pKa values in a single prose
|
||||
sentence. When matched, individual values are extracted and returned
|
||||
without temperature or label information.
|
||||
|
||||
Args:
|
||||
line: The raw input string to test.
|
||||
|
||||
Returns:
|
||||
A list of [`ParsedPKa`][data.dissociation_constant.ParsedPKa] records,
|
||||
one per numeric value found in the sentence, or `None` if this
|
||||
sentence form is not present in `line`.
|
||||
"""
|
||||
match = cls._PATTERN_MULTI_VALUE.search(line)
|
||||
if match is None:
|
||||
return None
|
||||
|
||||
nums = re.findall(r"-?[\d.]+", match.group(1))
|
||||
return [
|
||||
ParsedPKa(
|
||||
pka_label=None,
|
||||
pka_value=float(val),
|
||||
temperature_C=None,
|
||||
comment=None,
|
||||
)
|
||||
for val in nums
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def _try_patterns(cls, part: str, original_line: str) -> ParsedPKa | None:
|
||||
"""Try each compiled pattern against a single text segment.
|
||||
|
||||
Patterns are attempted in priority order. A match is only accepted
|
||||
when it captures a non-empty label group, which filters out numeric
|
||||
strings that are not actually pKa values (e.g. density or solubility
|
||||
values deposited under the wrong heading).
|
||||
|
||||
Args:
|
||||
part: A single semicolon-split segment of the original input,
|
||||
with leading/trailing whitespace and quotes stripped.
|
||||
original_line: The full original input line, retained verbatim
|
||||
for the `comment` field (with commas replaced by
|
||||
semicolons).
|
||||
|
||||
Returns:
|
||||
A [`ParsedPKa`][data.dissociation_constant.ParsedPKa] record if a pattern
|
||||
matches and a label is present, or `None` if no pattern yields
|
||||
a valid match.
|
||||
"""
|
||||
for pattern in cls._PATTERNS:
|
||||
match = pattern.search(part)
|
||||
if match is None:
|
||||
continue
|
||||
|
||||
groups = match.groupdict()
|
||||
label: str | None = groups.get("label") or None
|
||||
# Reject matches that did not capture a label: these are almost
|
||||
# always non-pKa numeric values (densities, concentrations, etc.)
|
||||
# that happen to match the loose fallback pattern.
|
||||
# See known false-positive examples:
|
||||
# CID 13343: "2.02 g/cu cm at 20 °C"
|
||||
# CID 2256: "1.1X10-4 at 25 °C"
|
||||
# CID 6101: "-1.34"
|
||||
if label is None:
|
||||
continue
|
||||
|
||||
temp_str: str | None = groups.get("temp")
|
||||
temperature: float | None = (
|
||||
float(temp_str) if temp_str is not None else None
|
||||
)
|
||||
|
||||
return ParsedPKa(
|
||||
pka_label=label,
|
||||
pka_value=float(match.group("value")),
|
||||
temperature_C=temperature,
|
||||
comment=original_line.replace(",", ";"),
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _parse_part(cls, part: str, original_line: str) -> ParsedPKa | None:
|
||||
"""Parse a single semicolon-split segment of a pKa string.
|
||||
|
||||
Delegates to
|
||||
[`_try_patterns`][data.dissociation_constant.DissociationConstantData._try_patterns]
|
||||
and logs a warning when no pattern matches, so that
|
||||
[`parse_value`][data.dissociation_constant.DissociationConstantData.parse_value]
|
||||
stays free of logging concerns.
|
||||
|
||||
Args:
|
||||
part: A single segment, already stripped of leading/trailing
|
||||
whitespace and surrounding quotes.
|
||||
original_line: The full original input line, passed through to
|
||||
[`_try_patterns`][data.dissociation_constant.DissociationConstantData._try_patterns]
|
||||
for provenance.
|
||||
|
||||
Returns:
|
||||
A [`ParsedPKa`][data.dissociation_constant.ParsedPKa] record, or
|
||||
`None` if the segment could not be matched to any known pKa format.
|
||||
"""
|
||||
result = cls._try_patterns(part, original_line)
|
||||
if result is None:
|
||||
logger.warning("Failed to parse segment: '{}'", part)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _extract_ids(
|
||||
entry: AnnotationEntry,
|
||||
) -> tuple[int, int] | None:
|
||||
"""Extract the CID and SID from a PubChem annotation entry.
|
||||
|
||||
Both identifiers must be present for the entry to be usable. If
|
||||
either is missing the entry is malformed and should be skipped.
|
||||
|
||||
Args:
|
||||
entry: A single annotation entry dict as returned by the
|
||||
PUG-View API.
|
||||
|
||||
Returns:
|
||||
A `(cid, sid)` tuple of integers, or `None` if either key
|
||||
is absent or the `CID` list is empty.
|
||||
"""
|
||||
try:
|
||||
cid = int(entry["LinkedRecords"]["CID"][0])
|
||||
sid = int(entry["SourceID"])
|
||||
return cid, sid
|
||||
except (KeyError, IndexError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_pclid(datum: PubChemDatum) -> int | None:
|
||||
"""Extract the PCLID from a single datum's ExtendedReference.
|
||||
|
||||
The PCLID (PubChem Live Data Identifier) links a specific measurement
|
||||
to its source record. It is optional: not all depositors provide it.
|
||||
|
||||
Args:
|
||||
datum: A single data point dict from a PubChem annotation entry.
|
||||
|
||||
Returns:
|
||||
The integer PCLID if present, or `None` if the key path does
|
||||
not exist.
|
||||
"""
|
||||
try:
|
||||
return int(datum["ExtendedReference"][0]["Matched"]["PCLID"])
|
||||
except (KeyError, IndexError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_string_value(datum: PubChemDatum) -> str | None:
|
||||
"""Extract the raw pKa string from a datum's Value field.
|
||||
|
||||
Args:
|
||||
datum: A single data point dict from a PubChem annotation entry.
|
||||
|
||||
Returns:
|
||||
The raw string value if present, or `None` if the expected
|
||||
key path does not exist or is empty.
|
||||
"""
|
||||
try:
|
||||
return datum["Value"]["StringWithMarkup"][0]["String"]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def parse_value(cls, line: str) -> list[ParsedPKa]:
|
||||
"""Parse a free-text pKa string into a list of structured records.
|
||||
|
||||
The input may contain one or more pKa values separated by
|
||||
semicolons, or a prose sentence listing multiple values. Each
|
||||
recognized value is returned as a
|
||||
[`ParsedPKa`][data.dissociation_constant.ParsedPKa]
|
||||
record. Segments that cannot be matched to any known format are logged at
|
||||
WARNING level and excluded from the output.
|
||||
|
||||
Args:
|
||||
line: A raw pKa string as deposited in PubChem, for example:
|
||||
|
||||
- `"pKa = 10.4 at 40 °C (tertiary amine)"`
|
||||
- `"pKa1 = 3.25; pKa2 = 4.76"`
|
||||
- `"pKa values are 3.25, 4.76, and 6.17"`
|
||||
- `"Weak acid. pK (25 °C): 3.35"`
|
||||
- `"pKa = 0.7 (caffeine cation)"`
|
||||
- `"pKa = 20"`
|
||||
|
||||
Returns:
|
||||
A list of [`ParsedPKa`][data.dissociation_constant.ParsedPKa] records,
|
||||
one per recognized pKa value. Returns an empty list if no values
|
||||
could be parsed.
|
||||
"""
|
||||
# Fast path: "pKa values are X, Y, and Z" prose sentence.
|
||||
multi = cls._parse_multi_value_sentence(line)
|
||||
if multi is not None:
|
||||
return multi
|
||||
|
||||
# General path: split on semicolons, parse each segment.
|
||||
parts = [p.strip().strip("'\"") for p in line.split(";") if p.strip()]
|
||||
results: list[ParsedPKa] = []
|
||||
for part in parts:
|
||||
record = cls._parse_part(part, line)
|
||||
if record is not None:
|
||||
results.append(record)
|
||||
return results
|
||||
|
||||
@classmethod
|
||||
def from_page(cls, annotation_data: list[AnnotationEntry]) -> pl.DataFrame:
|
||||
"""Convert a list of PubChem annotation entries into a tidy DataFrame.
|
||||
|
||||
Each entry in `annotation_data` corresponds to a single depositor
|
||||
record for one compound. This method extracts the compound and
|
||||
source identifiers, then parses every free-text pKa string within
|
||||
the entry into structured :class:`FlatPKaRecord` rows.
|
||||
|
||||
Entries with missing CID or SID are skipped with a WARNING log.
|
||||
Individual data points whose string value cannot be extracted or
|
||||
parsed are skipped with a WARNING log. Broad exception types are
|
||||
never swallowed: only specific, expected failure modes are handled.
|
||||
|
||||
Args:
|
||||
annotation_data: A list of annotation entry dicts as returned by
|
||||
[`get_data`][pubchem.PubChemAPI.get_data] for
|
||||
the `"Dissociation Constants"` heading. Each entry is
|
||||
expected to conform to [`AnnotationEntry`][responses.AnnotationEntry].
|
||||
|
||||
Returns:
|
||||
A polars DataFrame with one row per parsed pKa value
|
||||
and the following columns:
|
||||
|
||||
- `cid` (Int64): PubChem Compound Identifier.
|
||||
- `sid` (Int64): PubChem Substance Identifier.
|
||||
- `pclid` (Int64, nullable): PubChem Live Data Identifier.
|
||||
- `pka_label` (String, nullable): Ionisation-site label.
|
||||
- `pka_value` (Float64): The numeric pKa value.
|
||||
- `temperature_C` (Float64, nullable): Measurement temperature.
|
||||
- `comment` (String, nullable): Original source line.
|
||||
|
||||
Returns an empty DataFrame with the above schema when no valid
|
||||
rows could be extracted from `annotation_data`.
|
||||
|
||||
Raises:
|
||||
TypeError: If `annotation_data` is not a list.
|
||||
"""
|
||||
rows: Iterable[FlatPKaRecord] = []
|
||||
|
||||
for entry in annotation_data:
|
||||
logger.debug("Processing entry: {}", entry)
|
||||
|
||||
ids = cls._extract_ids(entry)
|
||||
if ids is None:
|
||||
logger.warning(
|
||||
"Skipping entry: could not extract CID and/or SID. Entry: {}",
|
||||
entry,
|
||||
)
|
||||
continue
|
||||
cid, sid = ids
|
||||
logger.debug("Identified CID {} from SID {}", cid, sid)
|
||||
|
||||
for datum in entry.get("Data", []):
|
||||
pclid = cls._extract_pclid(datum)
|
||||
|
||||
raw_string = cls._extract_string_value(datum)
|
||||
if raw_string is None:
|
||||
logger.warning(
|
||||
"Skipping datum for CID {}: could not extract string value. "
|
||||
"Datum: {}",
|
||||
cid,
|
||||
datum,
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
parsed_values = cls.parse_value(raw_string)
|
||||
except (ValueError, re.error) as exc:
|
||||
logger.warning(
|
||||
"Skipping datum for CID {}: parse_value raised {}: {}. "
|
||||
"Raw string: '{}'",
|
||||
cid,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
raw_string,
|
||||
)
|
||||
continue
|
||||
|
||||
for pka in parsed_values:
|
||||
rows.append(
|
||||
FlatPKaRecord(
|
||||
cid=cid,
|
||||
sid=sid,
|
||||
pclid=pclid,
|
||||
pka_label=pka["pka_label"],
|
||||
pka_value=pka["pka_value"],
|
||||
temperature_C=pka["temperature_C"],
|
||||
comment=pka["comment"],
|
||||
)
|
||||
)
|
||||
|
||||
if not rows:
|
||||
return pl.DataFrame(schema=_OUTPUT_SCHEMA)
|
||||
|
||||
return pl.from_dicts(rows)
|
||||
@@ -0,0 +1,18 @@
|
||||
from pcdigitizer.data import Annotation, AnnotationProcessor, DissociationConstantData
|
||||
|
||||
ANNOTATION_REGISTRY: dict[Annotation, type[AnnotationProcessor]] = {
|
||||
Annotation.DISSOCIATION_CONSTANTS: DissociationConstantData,
|
||||
}
|
||||
"""Maps PubChem annotation strings to their processor class."""
|
||||
|
||||
|
||||
def get_processor(annotation: Annotation) -> type[AnnotationProcessor]:
|
||||
"""Look up the processor for an annotation, or raise an error."""
|
||||
try:
|
||||
return ANNOTATION_REGISTRY[annotation]
|
||||
except KeyError:
|
||||
supported = ", ".join(sorted(h.value for h in ANNOTATION_REGISTRY))
|
||||
raise ValueError(
|
||||
f"No processor registered for '{annotation}'. "
|
||||
f"Supported annotations: {supported}"
|
||||
)
|
||||
@@ -0,0 +1,492 @@
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from urllib.parse import quote, urlencode, urljoin, urlparse
|
||||
|
||||
import polars as pl
|
||||
import requests
|
||||
from loguru import logger
|
||||
|
||||
from pcdigitizer.data import Annotation
|
||||
from pcdigitizer.responses import (
|
||||
AnnotationEntry,
|
||||
AnnotationsEnvelope,
|
||||
InformationListEnvelope,
|
||||
)
|
||||
|
||||
|
||||
class PubChemAPI:
|
||||
"""Client for the PubChem PUG-REST and PUG-View APIs.
|
||||
|
||||
All methods are class methods or static methods; no instance state is
|
||||
required. Network access is centralized in
|
||||
[`make_request`][pubchem.PubChemAPI.make_request] so that it can
|
||||
be replaced with a mock session during testing.
|
||||
"""
|
||||
|
||||
BASE_URL: str = "https://pubchem.ncbi.nlm.nih.gov/rest/"
|
||||
|
||||
ALLOWED_NAMESPACES: dict[str, set[str]] = {
|
||||
"compound": {
|
||||
"cid",
|
||||
"name",
|
||||
"smiles",
|
||||
"inchi",
|
||||
"inchikey",
|
||||
"formula",
|
||||
"listkey",
|
||||
},
|
||||
"substance": {"sid", "sourceid", "sourceall", "name", "xref", "listkey"},
|
||||
"assay": {"aid", "listkey", "type", "sourceall", "target", "activity"},
|
||||
"gene": {"geneid", "genesymbol", "synonym"},
|
||||
"protein": {"accession", "gi", "synonym"},
|
||||
"pathway": {"pwacc"},
|
||||
"taxonomy": {"taxid", "synonym"},
|
||||
"cell": {"cellacc", "synonym"},
|
||||
"annotations": {"sourcename", "headings", "heading"},
|
||||
}
|
||||
|
||||
_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"^[\w,\.\- ]+$")
|
||||
_VALID_PUG_ENDPOINTS: frozenset[str] = frozenset(("pug", "pug_view", "pug_soap"))
|
||||
|
||||
@staticmethod
|
||||
def make_request(url: str, session: requests.Session | None = None) -> bytes:
|
||||
"""Perform an HTTP GET and return the raw response body.
|
||||
|
||||
Args:
|
||||
url: The fully constructed PubChem REST URL to fetch.
|
||||
session: An optional [`Session`][requests.Session] to use for the
|
||||
request. When `None` the module-level `requests.get`
|
||||
function is used. Pass a session (or a mock) during testing
|
||||
to avoid live network calls.
|
||||
|
||||
Returns:
|
||||
The raw response body.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the server returns a non-200 HTTP status code.
|
||||
"""
|
||||
get = session.get if session is not None else requests.get
|
||||
response = get(url)
|
||||
if response.status_code == 200:
|
||||
logger.debug("Successful response for {}", url)
|
||||
return response.content
|
||||
raise RuntimeError(f"Request failed ({response.status_code}) for {url}")
|
||||
|
||||
@classmethod
|
||||
def make_json(
|
||||
cls, url: str, session: requests.Session | None = None
|
||||
) -> dict[str, object]:
|
||||
"""Fetch a URL and parse the response body as JSON.
|
||||
|
||||
Args:
|
||||
url: The fully constructed PubChem REST URL to fetch.
|
||||
session: An optional [`Session`][requests.Session] forwarded to
|
||||
[`make_request`][pubchem.PubChemAPI.make_request].
|
||||
See that method for details.
|
||||
|
||||
Returns:
|
||||
The top-level JSON object as a plain `dict`.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the HTTP request fails (propagated from
|
||||
[`make_request`][pubchem.PubChemAPI.make_request]).
|
||||
json.JSONDecodeError: If the response body is not valid JSON.
|
||||
"""
|
||||
text = cls.make_request(url, session=session)
|
||||
return json.loads(text)
|
||||
|
||||
@classmethod
|
||||
def _validate_url(cls, url: str) -> None:
|
||||
"""Verify that a URL is a safe, well-formed PubChem PUG endpoint.
|
||||
|
||||
Args:
|
||||
url: The URL string to validate.
|
||||
|
||||
Raises:
|
||||
ValueError: If the URL does not use HTTPS, does not point to
|
||||
`pubchem.ncbi.nlm.nih.gov`, or whose path does not start
|
||||
with `/rest/pug`.
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme != "https":
|
||||
raise ValueError(f"URL must use HTTPS; got scheme '{parsed.scheme}'.")
|
||||
if parsed.netloc != "pubchem.ncbi.nlm.nih.gov":
|
||||
raise ValueError(
|
||||
f"URL must point to pubchem.ncbi.nlm.nih.gov; got '{parsed.netloc}'."
|
||||
)
|
||||
if not parsed.path.startswith("/rest/pug"):
|
||||
raise ValueError(
|
||||
f"URL path must start with /rest/pug; got '{parsed.path}'."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _validate_components(
|
||||
cls, domain: str, namespace: str, identifiers: str
|
||||
) -> None:
|
||||
"""Validate the domain, namespace key, and identifier string.
|
||||
|
||||
The namespace may contain a `/` separator (e.g. `sourcename/ChEBI`);
|
||||
only the portion before the first `/` is checked against the allowed
|
||||
set for the given domain.
|
||||
|
||||
Args:
|
||||
domain: The PubChem domain (e.g. `"compound"`, `"annotations"`).
|
||||
namespace: The namespace string, optionally including a `/`-separated
|
||||
value (e.g. `"sourcename/ChEBI"`).
|
||||
identifiers: The identifier string to look up. May be empty only
|
||||
for the `"annotations"` domain.
|
||||
|
||||
Raises:
|
||||
ValueError: If `domain` is not in
|
||||
[`ALLOWED_NAMESPACES`][pubchem.PubChemAPI.ALLOWED_NAMESPACES].
|
||||
ValueError: If the namespace key is not valid for the given domain.
|
||||
ValueError: If `identifiers` is empty for a domain that requires it.
|
||||
ValueError: If `identifiers` contains characters outside the
|
||||
allowed set `[A-Za-z0-9_,.-_ ]`.
|
||||
"""
|
||||
if domain not in cls.ALLOWED_NAMESPACES:
|
||||
raise ValueError(
|
||||
f"Domain '{domain}' is not supported. "
|
||||
f"Choose from: {sorted(cls.ALLOWED_NAMESPACES)}."
|
||||
)
|
||||
|
||||
key = namespace.split("/", 1)[0]
|
||||
if key not in cls.ALLOWED_NAMESPACES[domain]:
|
||||
raise ValueError(
|
||||
f"Namespace '{key}' is not valid for domain '{domain}'. "
|
||||
f"Choose from: {sorted(cls.ALLOWED_NAMESPACES[domain])}."
|
||||
)
|
||||
|
||||
if domain != "annotations" and not identifiers:
|
||||
raise ValueError(f"Identifiers cannot be empty for domain '{domain}'.")
|
||||
|
||||
if identifiers and not cls._IDENTIFIER_PATTERN.match(identifiers):
|
||||
raise ValueError(
|
||||
f"Identifiers '{identifiers}' contain invalid characters. "
|
||||
"Only alphanumerics, commas, dots, hyphens, underscores, and "
|
||||
"spaces are permitted."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _encode_namespace(cls, namespace: str) -> tuple[str, str]:
|
||||
"""Split and percent-encode a namespace string.
|
||||
|
||||
A namespace may be a bare key (e.g. `"cid"`) or a key with a
|
||||
`/`-separated value (e.g. `"sourcename/ChEBI"`). Both segments
|
||||
are percent-encoded for safe inclusion in a URL path.
|
||||
|
||||
Args:
|
||||
namespace: The namespace string to encode, with an optional
|
||||
`/`-separated value component.
|
||||
|
||||
Returns:
|
||||
A two-tuple `(encoded_key, encoded_value)` where
|
||||
`encoded_value` is an empty string when no value component
|
||||
is present.
|
||||
"""
|
||||
if "/" in namespace:
|
||||
key, val = namespace.split("/", 1)
|
||||
else:
|
||||
key, val = namespace, ""
|
||||
return quote(key, safe=""), quote(val, safe="") if val else ""
|
||||
|
||||
@classmethod
|
||||
def _build_path(
|
||||
cls,
|
||||
domain: str,
|
||||
encoded_key: str,
|
||||
encoded_val: str,
|
||||
encoded_identifiers: str,
|
||||
operation: str | None,
|
||||
output_format: str,
|
||||
) -> str:
|
||||
"""Assemble the URL path segments into a single slash-joined string.
|
||||
|
||||
Args:
|
||||
domain: The PubChem domain (e.g. `"compound"`).
|
||||
encoded_key: The percent-encoded namespace key.
|
||||
encoded_val: The percent-encoded namespace value, or an empty
|
||||
string when absent.
|
||||
encoded_identifiers: The percent-encoded identifier string, or
|
||||
an empty string when absent.
|
||||
operation: An optional operation string (e.g. `"property/MolecularWeight"`).
|
||||
Slashes within this string are preserved as path separators.
|
||||
output_format: The desired output format (e.g. `"JSON"`, `"CSV"`).
|
||||
|
||||
Returns:
|
||||
A slash-joined path string ending with the output format segment,
|
||||
suitable for appending to a PUG base URL.
|
||||
"""
|
||||
parts: list[str] = [domain, encoded_key]
|
||||
if encoded_val:
|
||||
parts.append(encoded_val)
|
||||
if encoded_identifiers:
|
||||
parts.append(encoded_identifiers)
|
||||
if operation:
|
||||
parts.extend(operation.split("/"))
|
||||
parts.append(output_format)
|
||||
return "/".join(parts)
|
||||
|
||||
@classmethod
|
||||
def build_url(
|
||||
cls,
|
||||
domain: str,
|
||||
namespace: str,
|
||||
pug: str = "pug",
|
||||
identifiers: str = "",
|
||||
operation: str | None = None,
|
||||
output_format: str = "JSON",
|
||||
options: dict[str, str | int] | None = None,
|
||||
) -> str:
|
||||
"""Construct, validate, and return a fully formed PUG-REST URL.
|
||||
|
||||
Args:
|
||||
domain: The PubChem domain to query (e.g. `"compound"`,
|
||||
`"annotations"`). Must be a key in
|
||||
[`ALLOWED_NAMESPACES`][pubchem.PubChemAPI.ALLOWED_NAMESPACES].
|
||||
namespace: The namespace within the domain, optionally with a
|
||||
`/`-separated value (e.g. `"name"`, `"sourcename/ChEBI"`).
|
||||
pug: The PUG endpoint variant to use. Must be one of `"pug"`,
|
||||
`"pug_view"`, or `"pug_soap"`. Defaults to `"pug"`.
|
||||
identifiers: The record identifier(s) to look up, as a
|
||||
comma-separated string. May be empty for the
|
||||
`"annotations"` domain only.
|
||||
operation: An optional operation to perform on the matched
|
||||
records (e.g. `"property/MolecularWeight"`). Slashes are
|
||||
treated as path separators.
|
||||
output_format: The response format requested from PubChem.
|
||||
Defaults to `"JSON"`.
|
||||
options: Optional query-string parameters appended to the URL
|
||||
(e.g. `{"page": 2}`).
|
||||
|
||||
Returns:
|
||||
A fully constructed, validated HTTPS URL string ready to pass to
|
||||
[`make_request`][pubchem.PubChemAPI.make_request].
|
||||
|
||||
Raises:
|
||||
ValueError: If `pug` is not a recognized endpoint variant.
|
||||
ValueError: If `domain`, `namespace`, or `identifiers` fail
|
||||
validation (propagated from
|
||||
[`_validate_components`][pubchem.PubChemAPI._validate_components]).
|
||||
ValueError: If the resulting URL fails the safety check
|
||||
(propagated from [`_validate_url`][pubchem.PubChemAPI._validate_url]).
|
||||
"""
|
||||
if pug not in cls._VALID_PUG_ENDPOINTS:
|
||||
raise ValueError(
|
||||
f"Invalid PUG endpoint '{pug}'. "
|
||||
f"Choose from: {sorted(cls._VALID_PUG_ENDPOINTS)}."
|
||||
)
|
||||
|
||||
cls._validate_components(domain, namespace, identifiers)
|
||||
|
||||
encoded_key, encoded_val = cls._encode_namespace(namespace)
|
||||
encoded_identifiers = quote(identifiers, safe=",") if identifiers else ""
|
||||
|
||||
path = cls._build_path(
|
||||
domain,
|
||||
encoded_key,
|
||||
encoded_val,
|
||||
encoded_identifiers,
|
||||
operation,
|
||||
output_format,
|
||||
)
|
||||
|
||||
pug_base = pug if pug.endswith("/") else pug + "/"
|
||||
url = urljoin(cls.BASE_URL, pug_base)
|
||||
url = urljoin(url, path)
|
||||
|
||||
if options:
|
||||
qs = urlencode(options, doseq=True, quote_via=quote)
|
||||
url = f"{url}?{qs}"
|
||||
|
||||
cls._validate_url(url)
|
||||
return url
|
||||
|
||||
@classmethod
|
||||
def get_sources(cls, session: requests.Session | None = None) -> pl.DataFrame:
|
||||
"""Fetch the full PubChem depositor source table as a DataFrame.
|
||||
|
||||
Retrieves the CSV source table from
|
||||
`/rest/pug/sourcetable/all/CSV`, which lists every organization
|
||||
that has deposited data into PubChem along with associated metadata.
|
||||
|
||||
Args:
|
||||
session: An optional [`Session`][requests.Session] forwarded to
|
||||
[`make_request`][pubchem.PubChemAPI.make_request]. Pass a mock during
|
||||
testing to avoid live network calls.
|
||||
|
||||
Returns:
|
||||
A `polars.DataFrame` with one row per depositor source.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the HTTP request fails.
|
||||
"""
|
||||
url = urljoin(cls.BASE_URL, "pug/sourcetable/all/CSV")
|
||||
raw = cls.make_request(url, session=session)
|
||||
return pl.read_csv(raw)
|
||||
|
||||
@staticmethod
|
||||
def _process_annotations(
|
||||
raw_annotations: list[AnnotationEntry],
|
||||
) -> dict[str, list[str]]:
|
||||
"""Group annotation names by their type.
|
||||
|
||||
Args:
|
||||
raw_annotations: A list of annotation records as returned by the
|
||||
PubChem API, each containing at minimum a `"Type"` key and
|
||||
a `"Heading"` key.
|
||||
|
||||
Returns:
|
||||
A dict mapping each annotation type (e.g. `"Compound"`) to a
|
||||
list of heading names belonging to that type, in the order they
|
||||
were encountered.
|
||||
"""
|
||||
grouped: dict[str, list[str]] = defaultdict(list)
|
||||
for entry in raw_annotations:
|
||||
grouped[entry["Type"]].append(entry["Heading"])
|
||||
return grouped
|
||||
|
||||
@classmethod
|
||||
def get_source_annotations(
|
||||
cls,
|
||||
source_name: str,
|
||||
session: requests.Session | None = None,
|
||||
) -> dict[str, list[str]]:
|
||||
"""Fetch all annotation headings deposited by a specific source.
|
||||
|
||||
Retrieves annotations from the `annotations/sourcename/<source>`
|
||||
endpoint and groups them by type via
|
||||
[`_process_annotations`][pubchem.PubChemAPI._process_annotations].
|
||||
|
||||
Note:
|
||||
`output_format` is not exposed as a parameter here because
|
||||
[`make_json`][pubchem.PubChemAPI.make_json] always expects a JSON response.
|
||||
To retrieve raw non-JSON data from this endpoint, use
|
||||
[`build_url`][pubchem.PubChemAPI.build_url] and
|
||||
[`make_request`][pubchem.PubChemAPI.make_request] directly.
|
||||
|
||||
Args:
|
||||
source_name: The PubChem depositor source name to query
|
||||
(e.g. `"ChEBI"`). Forward slashes are replaced with
|
||||
periods as required by the PubChem API.
|
||||
session: An optional [`Session`][requests.Session] forwarded to
|
||||
[`make_request`][pubchem.PubChemAPI.make_request]. Pass a mock during
|
||||
testing to avoid live network calls.
|
||||
|
||||
Returns:
|
||||
A dict mapping annotation type strings (e.g. `"Compound"`) to
|
||||
lists of heading names provided by the given source.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the HTTP request fails.
|
||||
KeyError: If the response does not contain the expected
|
||||
`InformationList.Annotation` structure.
|
||||
"""
|
||||
safe_source = source_name.replace("/", ".")
|
||||
namespace = f"sourcename/{safe_source}"
|
||||
|
||||
url = cls.build_url(
|
||||
domain="annotations",
|
||||
namespace=namespace,
|
||||
identifiers="",
|
||||
operation=None,
|
||||
output_format="JSON",
|
||||
)
|
||||
raw: InformationListEnvelope = cls.make_json(
|
||||
url, session=session
|
||||
) # ty:ignore[invalid-assignment]
|
||||
raw_annotations: list[AnnotationEntry] = raw["InformationList"]["Annotation"]
|
||||
return cls._process_annotations(raw_annotations)
|
||||
|
||||
@classmethod
|
||||
def get_annotations(
|
||||
cls, session: requests.Session | None = None
|
||||
) -> dict[str, list[str]]:
|
||||
"""Retrieve all annotation headings available in PubChem.
|
||||
|
||||
Fetches and processes the results of
|
||||
`/rest/pug/annotations/headings/JSON`. The returned dict normally
|
||||
contains the following type keys: `Assay`, `Cell`, `Compound`,
|
||||
`Element`, `Gene`, `Pathway`, `Protein`, `Taxonomy`.
|
||||
|
||||
Args:
|
||||
session: An optional [`Session`][requests.Session] forwarded to
|
||||
[`make_request`][pubchem.PubChemAPI.make_request]. Pass a mock during
|
||||
testing to avoid live network calls.
|
||||
|
||||
Returns:
|
||||
A dict mapping annotation type strings to lists of heading names
|
||||
belonging to that type.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the HTTP request fails.
|
||||
KeyError: If the response does not contain the expected
|
||||
`InformationList.Annotation` structure.
|
||||
"""
|
||||
url = cls.build_url(
|
||||
domain="annotations",
|
||||
namespace="headings",
|
||||
identifiers="",
|
||||
operation=None,
|
||||
output_format="JSON",
|
||||
)
|
||||
raw: InformationListEnvelope = cls.make_json(
|
||||
url, session=session
|
||||
) # ty:ignore[invalid-assignment]
|
||||
raw_annotations: list[AnnotationEntry] = raw["InformationList"]["Annotation"]
|
||||
return cls._process_annotations(raw_annotations)
|
||||
|
||||
@classmethod
|
||||
def get_data(
|
||||
cls,
|
||||
annotation: Annotation,
|
||||
page: int | None = None,
|
||||
session: requests.Session | None = None,
|
||||
) -> list[AnnotationEntry]:
|
||||
"""Retrieve all records for a specific annotation.
|
||||
|
||||
Fetches data from the PUG-View `annotations/heading/<heading>`
|
||||
endpoint. Without a `page` argument this returns every result,
|
||||
which can be slow for popular headings.
|
||||
|
||||
Args:
|
||||
annotation: The PubChem annotation heading to download
|
||||
(e.g. `Annotation.DISSOCIATION_CONSTANTS`).
|
||||
page: If provided, fetch only this specific page of results.
|
||||
Must be a positive integer. If `None`, all results are
|
||||
returned.
|
||||
session: An optional [`Session`][requests.Session]Session` forwarded to
|
||||
[`make_request`][pubchem.PubChemAPI.make_request]. Pass a mock during
|
||||
testing to avoid live network calls.
|
||||
|
||||
Returns:
|
||||
A list of [`AnnotationEntry`][responses.AnnotationEntry] dicts for the
|
||||
requested data, in the order returned by the API.
|
||||
|
||||
Raises:
|
||||
ValueError: If `page` is provided but is less than 1.
|
||||
RuntimeError: If the HTTP request fails.
|
||||
KeyError: If the response does not contain the expected
|
||||
Annotations structure.
|
||||
"""
|
||||
if page is not None and page < 1:
|
||||
raise ValueError(f"page must be a positive integer; got {page}.")
|
||||
|
||||
options: dict[str, str | int] | None = (
|
||||
{"page": page} if page is not None else None
|
||||
)
|
||||
|
||||
url = cls.build_url(
|
||||
pug="pug_view",
|
||||
domain="annotations",
|
||||
namespace="heading",
|
||||
identifiers=annotation,
|
||||
operation=None,
|
||||
output_format="JSON",
|
||||
options=options,
|
||||
)
|
||||
raw: AnnotationsEnvelope = cls.make_json(
|
||||
url, session=session
|
||||
) # ty:ignore[invalid-assignment]
|
||||
return raw["Annotations"]["Annotation"]
|
||||
@@ -0,0 +1,275 @@
|
||||
"""`TypedDict` definitions for PubChem PUG-REST and PUG-View API responses.
|
||||
|
||||
This module provides a single source of truth for the JSON response shapes
|
||||
returned by the PubChem REST APIs.
|
||||
|
||||
## Structure overview
|
||||
|
||||
PubChem responses follow two broad envelope shapes depending on the endpoint:
|
||||
|
||||
`InformationList` envelope (PUG-REST `/annotations/headings` and
|
||||
`/annotations/sourcename`):
|
||||
|
||||
```json
|
||||
{
|
||||
"InformationList": {
|
||||
"Annotation": [ ... ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`Annotations` envelope (PUG-View `/annotations/heading`):
|
||||
|
||||
```json
|
||||
{
|
||||
"Annotations": {
|
||||
"Annotation": [ ... ],
|
||||
"Page": 1,
|
||||
"TotalPages": 42
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Within both envelopes each `Annotation` element is an
|
||||
[`AnnotationEntry`][responses.AnnotationEntry]. When the entry was fetched via
|
||||
PUG-View (i.e. a specific heading), it additionally carries a `Data` list whose
|
||||
elements are [`PubChemDatum`][responses.PubChemDatum] records containing the
|
||||
deposited values.
|
||||
|
||||
## Hierarchy
|
||||
|
||||
The nesting from outermost to innermost is:
|
||||
|
||||
```text
|
||||
InformationListEnvelope / AnnotationsEnvelope
|
||||
└── AnnotationEntry
|
||||
├── LinkedRecords
|
||||
└── Data
|
||||
└── PubChemDatum
|
||||
├── DatumValue
|
||||
│ └── StringWithMarkup
|
||||
└── ExtendedReference
|
||||
└── MatchedRecord
|
||||
```
|
||||
|
||||
All classes use `total=False` where keys are optional in the real API
|
||||
responses, and `total=True` (the default) where keys are always present.
|
||||
"""
|
||||
|
||||
from typing import TypedDict
|
||||
|
||||
|
||||
class MatchedRecord(TypedDict, total=False):
|
||||
"""The `Matched` sub-object within an
|
||||
[`ExtendedReference`][responses.ExtendedReference] entry.
|
||||
|
||||
Carries identifiers that link a specific deposited value to its
|
||||
corresponding record in PubChem's live data system.
|
||||
"""
|
||||
|
||||
PCLID: int
|
||||
"""
|
||||
PubChem Live Data Identifier. Links the measurement to its canonical record.
|
||||
Not present for all depositors.
|
||||
"""
|
||||
|
||||
|
||||
class ExtendedReference(TypedDict, total=False):
|
||||
"""A single element of the `ExtendedReference` list on a datum.
|
||||
|
||||
Provides cross-references from a deposited data point to other PubChem
|
||||
record types. The `Matched` key is present only when PubChem has
|
||||
successfully linked the value to a live data record.
|
||||
"""
|
||||
|
||||
Matched: MatchedRecord
|
||||
"""The matched record containing the PCLID, if available."""
|
||||
|
||||
|
||||
class StringWithMarkup(TypedDict):
|
||||
"""A single element of the `StringWithMarkup` list on a datum value.
|
||||
|
||||
PubChem wraps all deposited string values in this container, which in
|
||||
the full API response can also carry markup annotations (bold, italic,
|
||||
subscript, etc.). Only the plain string is modelled here since markup
|
||||
is not used by this package.
|
||||
"""
|
||||
|
||||
String: str
|
||||
"""The plain text content of the deposited value."""
|
||||
|
||||
|
||||
class DatumValue(TypedDict, total=False):
|
||||
"""The `Value` object on a single data point within an annotation.
|
||||
|
||||
PubChem data points can carry several value types (numeric, boolean, binary).
|
||||
This class models only the [`StringWithMarkup`][responses.StringWithMarkup]
|
||||
variant, which is the form used for textual property data such as pKa strings.
|
||||
"""
|
||||
|
||||
StringWithMarkup: list[StringWithMarkup]
|
||||
"""
|
||||
A list of string value containers. In practice this list contains exactly one
|
||||
element for property annotations.
|
||||
"""
|
||||
|
||||
|
||||
class PubChemDatum(TypedDict, total=False):
|
||||
"""A single data point within a PubChem PUG-View annotation entry.
|
||||
|
||||
Each [`AnnotationEntry`][responses.AnnotationEntry] fetched from a specific heading
|
||||
contains a `Data` list whose elements are `PubChemDatum` records. Each datum
|
||||
represents one deposited measurement or value from a single source.
|
||||
"""
|
||||
|
||||
Value: DatumValue
|
||||
"""
|
||||
The deposited value, in [`StringWithMarkup`][responses.StringWithMarkup]
|
||||
form for textual properties.
|
||||
"""
|
||||
|
||||
ExtendedReference: list[ExtendedReference]
|
||||
"""
|
||||
Cross-references linking this datum to other PubChem record types.
|
||||
Used to recover the PCLID when present.
|
||||
"""
|
||||
|
||||
|
||||
class LinkedRecords(TypedDict, total=False):
|
||||
"""The `LinkedRecords` object on a PubChem annotation entry.
|
||||
|
||||
Maps PubChem record type names to lists of integer identifiers for all
|
||||
records that are linked to this annotation entry. Additional record
|
||||
types beyond those listed here (e.g. `SID`, `AID`) may be present
|
||||
in real responses but are not modelled here.
|
||||
"""
|
||||
|
||||
CID: list[int]
|
||||
"""
|
||||
List of PubChem Compound Identifiers linked to this entry. Typically contains
|
||||
exactly one element for compound annotations.
|
||||
"""
|
||||
|
||||
SID: list[int]
|
||||
"""List of PubChem Substance Identifiers linked to this entry."""
|
||||
|
||||
AID: list[int]
|
||||
"""List of PubChem Assay Identifiers linked to this entry."""
|
||||
|
||||
|
||||
class AnnotationEntry(TypedDict, total=False):
|
||||
"""A single annotation record returned by the PubChem PUG-REST or
|
||||
PUG-View API.
|
||||
|
||||
This type covers both the lightweight form returned by
|
||||
`/annotations/headings` (which carries only `Type` and `Heading`)
|
||||
and the full form returned by `/annotations/heading/<heading>` (which
|
||||
additionally carries `Data`, `LinkedRecords`, `SourceID`, etc.).
|
||||
|
||||
Using `total=False` throughout reflects that the set of keys present
|
||||
varies significantly by endpoint and depositor.
|
||||
"""
|
||||
|
||||
SourceName: str
|
||||
"""The human-readable name of the depositing organization."""
|
||||
|
||||
SourceID: str
|
||||
"""
|
||||
The depositor's own identifier for this record (e.g. a database accession
|
||||
number). Cast to `int` where a numeric ID is expected.
|
||||
"""
|
||||
|
||||
Name: str
|
||||
"""The name of the compound or entity this annotation describes."""
|
||||
|
||||
Description: list[str]
|
||||
"""Free-text description lines provided by the depositor."""
|
||||
|
||||
Reference: list[str]
|
||||
"""Citation strings provided by the depositor."""
|
||||
|
||||
ANID: int
|
||||
"""
|
||||
PubChem Annotation Identifier, a unique integer assigned to this annotation record.
|
||||
"""
|
||||
|
||||
LinkedRecords: LinkedRecords
|
||||
"""
|
||||
Identifiers linking this entry to PubChem compound, substance, and assay records.
|
||||
"""
|
||||
|
||||
Type: str
|
||||
"""
|
||||
The broad category of this annotation heading (e.g. `"Compound"`,
|
||||
`"Gene"`, `"Assay"`).
|
||||
"""
|
||||
|
||||
Heading: str
|
||||
"""
|
||||
The specific annotation heading name (e.g. `"Dissociation Constants"`,
|
||||
`"Boiling Point"`).
|
||||
"""
|
||||
|
||||
URL: str
|
||||
"""A URL provided by the depositor pointing to the original source."""
|
||||
|
||||
Data: list[PubChemDatum]
|
||||
"""
|
||||
The list of individual deposited data points. Present only for entries fetched
|
||||
via a specific heading endpoint.
|
||||
"""
|
||||
|
||||
|
||||
class _InformationList(TypedDict, total=False):
|
||||
"""The inner `InformationList` object in a PUG-REST response."""
|
||||
|
||||
Annotation: list[AnnotationEntry]
|
||||
"""The list of annotation records."""
|
||||
|
||||
Page: int
|
||||
"""The current page number when paginated results are returned."""
|
||||
|
||||
TotalPages: int
|
||||
"""The total number of available pages."""
|
||||
|
||||
|
||||
class InformationListEnvelope(TypedDict):
|
||||
"""Top-level envelope for PUG-REST responses that wrap an InformationList.
|
||||
|
||||
Returned by endpoints such as:
|
||||
|
||||
- `/rest/pug/annotations/headings/JSON`
|
||||
- `/rest/pug/annotations/sourcename/<source>/JSON`
|
||||
"""
|
||||
|
||||
InformationList: _InformationList
|
||||
"""
|
||||
The inner object containing the annotation list and optional pagination metadata.
|
||||
"""
|
||||
|
||||
|
||||
class _AnnotationsList(TypedDict, total=False):
|
||||
"""The inner `Annotations` object in a PUG-View response."""
|
||||
|
||||
Annotation: list[AnnotationEntry]
|
||||
"""The list of annotation records."""
|
||||
|
||||
Page: int
|
||||
"""The current page number when paginated results are returned."""
|
||||
|
||||
TotalPages: int
|
||||
"""The total number of available pages."""
|
||||
|
||||
|
||||
class AnnotationsEnvelope(TypedDict):
|
||||
"""Top-level envelope for PUG-View responses that wrap an Annotations list.
|
||||
|
||||
Returned by endpoints such as:
|
||||
|
||||
- `/rest/pug_view/annotations/heading/<heading>/JSON`
|
||||
"""
|
||||
|
||||
Annotations: _AnnotationsList
|
||||
"""
|
||||
The inner object containing the annotation list and optional pagination metadata.
|
||||
"""
|
||||
@@ -0,0 +1,26 @@
|
||||
import polars as pl
|
||||
from raygent import Task
|
||||
|
||||
from pcdigitizer import PubChemAPI
|
||||
from pcdigitizer.data import Annotation, get_processor
|
||||
|
||||
|
||||
class GetAnnotationPage(Task):
|
||||
"""Get a page of annotated data from PubChem."""
|
||||
|
||||
def do(self, item: int, annotation: Annotation, **kwargs) -> pl.DataFrame:
|
||||
"""Download and process a single page of data from an annotation.
|
||||
|
||||
Args:
|
||||
item: Which page to fetch from `annotation`.
|
||||
annotation: What data to fetch.
|
||||
|
||||
Returns:
|
||||
Processed data in a polars DataFrame.
|
||||
"""
|
||||
session = kwargs.get("session", None)
|
||||
processor = get_processor(annotation)
|
||||
|
||||
annotation_data = PubChemAPI.get_data(annotation, item, session=session)
|
||||
df = processor.from_page(annotation_data)
|
||||
return df
|
||||
@@ -0,0 +1,83 @@
|
||||
[workspace]
|
||||
name = "pcdigitizer"
|
||||
authors = [
|
||||
"Scientific Computing Studio <us@scient.ing>",
|
||||
]
|
||||
channels = ["conda-forge"]
|
||||
description = "Turn raw PubChem data into clean, ML-ready chemical datasets"
|
||||
platforms = ["linux-64", "osx-64", "osx-arm64"]
|
||||
readme = "README.md"
|
||||
version = "26.4.10"
|
||||
|
||||
[pypi-dependencies]
|
||||
pcdigitizer = { path = ".", editable = true }
|
||||
raygent = { git = "https://github.com/scienting/raygent", branch = "main" }
|
||||
|
||||
[environments]
|
||||
dev = ["dev"]
|
||||
docs = ["docs"]
|
||||
|
||||
[tasks]
|
||||
|
||||
[dependencies]
|
||||
python = ">=3.13"
|
||||
loguru = ">=0.7.3,<0.8"
|
||||
beautifulsoup4 = ">=4.13.4,<5"
|
||||
polars = ">=1.30.0,<2"
|
||||
|
||||
[feature.dev.dependencies]
|
||||
ruff = ">=0.11.10,<0.12"
|
||||
ty = ">=0.0.25,<0.1.0"
|
||||
setuptools = ">=82.0.0,<83"
|
||||
setuptools-scm = ">=9.2.2,<10"
|
||||
wheel = ">=0.46.3,<0.47"
|
||||
pytest = ">=9.0.2,<10"
|
||||
pytest-cov = ">=4.1.0"
|
||||
coverage = ">=7.3.1"
|
||||
twine = ">=6.1.0,<7"
|
||||
bump-my-version = ">=1.2.7,<2"
|
||||
|
||||
[feature.dev.tasks]
|
||||
lint = { cmd = ["ruff", "check", "--fix", "--config", ".ruff.toml", "./"] }
|
||||
format = { depends-on = ["lint"], cmd = ["ruff", "check", "--select", "I", "--fix", "&&", "ruff", "format", "--config", ".ruff.toml", "./"] }
|
||||
tests = { cmd = [
|
||||
"PYTHONPATH=.",
|
||||
"pytest",
|
||||
"-c",
|
||||
".pytest.ini",
|
||||
"--cov='pcdigitizer'",
|
||||
"--cov-report=xml",
|
||||
"--junit-xml=report.xml",
|
||||
"--failed-first",
|
||||
]}
|
||||
coverage = { cmd = ["coverage", "report"] }
|
||||
cleanup-build = { cmd = ["rm", "-rf", "./build"] }
|
||||
bump = { cmd = ["bump-my-version", "bump", "patch"] }
|
||||
build = { cmd = ["python3", "-m", "build"], depends-on = ["cleanup-build"]}
|
||||
publish-test = { cmd = ["twine", "upload", "--repository", "testpypi", "dist/*"] }
|
||||
publish = { cmd = ["twine", "upload", "dist/*"] }
|
||||
|
||||
[feature.dev.pypi-dependencies]
|
||||
build = ">=1.2.2.post1,<2"
|
||||
setuptools-scm = ">=8.0.0"
|
||||
|
||||
[feature.docs.dependencies]
|
||||
# Will need to migrate to https://zensical.org/ once it matures.
|
||||
mkdocs = ">=1.6.1,<2"
|
||||
mkdocs-material = ">=9.5.44,<10"
|
||||
mkdocstrings = ">=1.0.3,<2"
|
||||
mkdocstrings-python = ">=2.0.2,<3"
|
||||
mkdocs-macros-plugin = ">=1.5.0,<2"
|
||||
mkdocs-glightbox = ">=0.5.2,<0.6"
|
||||
mkdocs-git-revision-date-localized-plugin = ">=1.2.9,<2"
|
||||
mkdocs-autorefs = ">=1.4.4,<2"
|
||||
mkdocs-gen-files = ">=0.6.0,<0.7"
|
||||
pymdown-extensions = ">=10.21,<11"
|
||||
super-collections = ">=0.6.2,<0.7"
|
||||
|
||||
[feature.docs.tasks]
|
||||
docs = { cmd = ["rm", "-rf", "./public/", "&&", "mkdocs", "build", "-d", "public/"] }
|
||||
docs-serve = { cmd = ["mkdocs", "serve"] }
|
||||
|
||||
[feature.docs.pypi-dependencies]
|
||||
mkdocs-awesome-nav = ">=3.3.0, <4"
|
||||
@@ -0,0 +1,31 @@
|
||||
[project]
|
||||
name = "pcdigitizer"
|
||||
authors = [
|
||||
{name = "Scientific Computing Studio", email = "us@scient.ing"}
|
||||
]
|
||||
maintainers = [
|
||||
{name = "Alex M. Maldonado", email = "alex@scient.ing"}
|
||||
]
|
||||
description = "Turn raw PubChem data into clean, ML-ready chemical datasets"
|
||||
dynamic = ["version"]
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.13"
|
||||
|
||||
dependencies = [
|
||||
"loguru>=0.7.2,<1.0.0",
|
||||
"beautifulsoup4>=4.13.4,<5",
|
||||
"polars>=1.30.0,<2",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=80", "setuptools-scm>=8", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["pcdigitizer*"]
|
||||
|
||||
[tool.setuptools_scm]
|
||||
write_to = "pcdigitizer/_version.py"
|
||||
version_scheme = "guess-next-dev"
|
||||
local_scheme = "node-and-timestamp"
|
||||
@@ -0,0 +1,47 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from pcdigitizer import enable_logging
|
||||
|
||||
TEST_DIR: Path = Path(__file__).resolve().parent
|
||||
TMP_DIR: Path = TEST_DIR / "tmp"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def turn_on_logging() -> None:
|
||||
"""Enable loguru logging at DEBUG level for the entire test session."""
|
||||
enable_logging(logging.DEBUG)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_dir() -> Path:
|
||||
"""Return the absolute path to the tests/ directory."""
|
||||
return TEST_DIR
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_dir() -> Path:
|
||||
"""Return the path to the shared temporary output directory."""
|
||||
TMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
return TMP_DIR
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_session(test_dir):
|
||||
"""A fake requests.Session that returns saved fixture files."""
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, content: bytes, status_code: int = 200):
|
||||
self.content = content
|
||||
self.status_code = status_code
|
||||
|
||||
class FakeSession:
|
||||
def get(self, url):
|
||||
if "heading/Dissociation" in url:
|
||||
fixture = test_dir / "files" / "dissociation-constants-1.json"
|
||||
return FakeResponse(fixture.read_bytes())
|
||||
return FakeResponse(b'{"Fault": "not found"}', status_code=404)
|
||||
|
||||
return FakeSession()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,33 @@
|
||||
import polars as pl
|
||||
|
||||
from pcdigitizer import Annotation, GetAnnotationPage
|
||||
|
||||
|
||||
def test_process_dissociation_data(test_dir, mock_session):
|
||||
"""Makes request for the first page of the `"Dissociation Constants"` and
|
||||
processes the data into a polars DataFrame.
|
||||
|
||||
Makes the following request:
|
||||
[pubchem.ncbi.nlm.nih.gov/rest/pug_view/annotations/heading/Dissociation%20Constants/JSON?page=1](https://pubchem.ncbi.nlm.nih.gov/rest/pug_view/annotations/heading/Dissociation%20Constants/JSON?page=1).
|
||||
"""
|
||||
task = GetAnnotationPage()
|
||||
df = task.do(
|
||||
item=1, annotation=Annotation.DISSOCIATION_CONSTANTS, session=mock_session
|
||||
)
|
||||
|
||||
# TODO: Need to implement actual tests
|
||||
assert not df.is_empty()
|
||||
assert df.columns == [
|
||||
"cid",
|
||||
"sid",
|
||||
"pclid",
|
||||
"pka_label",
|
||||
"pka_value",
|
||||
"temperature_C",
|
||||
"comment",
|
||||
]
|
||||
assert df.schema["pka_value"] == pl.Float64
|
||||
assert df.schema["cid"] == pl.Int64
|
||||
|
||||
csv_path = test_dir / "tmp" / "test_process_dissociation_data.csv"
|
||||
df.write_csv(csv_path)
|
||||
@@ -0,0 +1,48 @@
|
||||
import polars as pl
|
||||
|
||||
from pcdigitizer import Annotation, PubChemAPI
|
||||
|
||||
|
||||
def test_source_df():
|
||||
df = PubChemAPI.get_sources()
|
||||
|
||||
columns = df.columns
|
||||
assert len(columns) == 23
|
||||
assert columns[0] == "Source Name"
|
||||
assert columns[-3] == "License URL"
|
||||
|
||||
source_name = "Hazardous Substances Data Bank"
|
||||
source_count = df.select(
|
||||
(pl.col("Source Name").str.contains(source_name)).sum()
|
||||
).item()
|
||||
assert source_count == 1
|
||||
|
||||
|
||||
def test_source_annotations():
|
||||
annotations = PubChemAPI.get_source_annotations(
|
||||
"Hazardous Substances Data Bank (HSDB)"
|
||||
)
|
||||
assert len(annotations["Compound"]) == 246
|
||||
|
||||
|
||||
def test_annotations():
|
||||
annotations = PubChemAPI.get_annotations()
|
||||
annotation_types = sorted(annotations.keys())
|
||||
assert annotation_types == [
|
||||
"Assay",
|
||||
"Cell",
|
||||
"Compound",
|
||||
"Element",
|
||||
"Gene",
|
||||
"Pathway",
|
||||
"Protein",
|
||||
"Taxonomy",
|
||||
]
|
||||
assert "Dissociation Constants" in annotations["Compound"]
|
||||
|
||||
|
||||
def test_get_data(mock_session):
|
||||
data = PubChemAPI.get_data(
|
||||
Annotation.DISSOCIATION_CONSTANTS, 1, session=mock_session
|
||||
)
|
||||
assert len(data) == 1000
|
||||
@@ -0,0 +1,2 @@
|
||||
*
|
||||
!.gitignore
|
||||
Reference in New Issue
Block a user