donaldrich
/
jupyter-notebooks
1
0
Fork 0
Code Pull Requests Packages Projects Releases Wiki Activity

added furl

Browse Source
master
Don Aldrich 3 years ago
parent a052e77a8f
commit e206c89868
20 changed files with 4616 additions and 257 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      .gitignore
  2. BIN
      data_converion/__pycache__/tomd.cpython-310.pyc
  3. 87
      data_converion/tomd.ipynb
  4. 5
      data_converion/tomd.py
  5. 6
      data_transfer/Add Container Inventory to Directus.ipynb
  6. 2
      data_transfer/Add Inventories to wiki.ipynb
  7. 2
      data_transfer/Add Linkace Inventory to Directus.ipynb
  8. 2
      data_transfer/Add traefic routers.ipynb
  9. 640
      documentation/python-cheatsheet.ipynb
  10. 2485
      documentation/terraform-cheatsheet.ipynb
  11. 119
      documentation/terraform-docs.ipynb
  12. 18
      documentation/weasyprint.ipynb
  13. 287
      http/furl/furl.ipynb
  14. 784
      http/furl/readme.md
  15. 126
      playground/nlp/spacy.ipynb
  16. 53
      storage/minio.ipynb
  17. 26
      templating/Dashmachine Config Template.ipynb
  18. 15
      webscraping/pyppeteer/walgreens.ipynb
  19. 174
      webscraping/rendering/splash.ipynb
  20. 40
      webscraping/robot_framework/Untitled.ipynb

2
.gitignore vendored
Unescape View File

@ -1,2 +1,4 @@
unpublished/
.DS_Store
.env
temp/

BIN
data_converion/__pycache__/tomd.cpython-310.pyc
View File

Binary file not shown.

87
data_converion/tomd.ipynb
Unescape View File

@ -0,0 +1,87 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Defaulting to user installation because normal site-packages is not writeable\n",
"Requirement already satisfied: tomd in /home/donaldrich/.local/lib/python3.10/site-packages (0.1.3)\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
}
],
"source": [
"%pip install tomd"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"```python\n",
"import tomd\n",
"\n",
"tomd.Tomd('<h1>h1</h1>').markdown\n",
"# or\n",
"tomd.convert('<h1>h1</h1>')\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "partially initialized module 'tomd' has no attribute 'Tomd' (most likely due to a circular import)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/home/donaldrich/projects/secrets/jupyter-notebooks/data_converion/tomd.ipynb Cell 2\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/data_converion/tomd.ipynb#W0sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mtomd\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/data_converion/tomd.ipynb#W0sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39m# from tomd import Tomd\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/data_converion/tomd.ipynb#W0sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m \u001b[39m# tomd.Tomd('<h1>h1</h1>').markdown\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/data_converion/tomd.ipynb#W0sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3'>4</a>\u001b[0m \u001b[39m# or\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/data_converion/tomd.ipynb#W0sdnNjb2RlLXJlbW90ZQ%3D%3D?line=4'>5</a>\u001b[0m tomd\u001b[39m.\u001b[39mconvert(\u001b[39m'\u001b[39m\u001b[39m<h1>h1</h1>\u001b[39m\u001b[39m'\u001b[39m)\n",
"File \u001b[0;32m~/projects/secrets/jupyter-notebooks/data_converion/tomd.py:3\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mtomd\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m tomd\u001b[39m.\u001b[39;49mTomd(\u001b[39m'\u001b[39m\u001b[39m<h1>h1</h1>\u001b[39m\u001b[39m'\u001b[39m)\u001b[39m.\u001b[39mmarkdown\n\u001b[1;32m 4\u001b[0m \u001b[39m# or\u001b[39;00m\n\u001b[1;32m 5\u001b[0m tomd\u001b[39m.\u001b[39mconvert(\u001b[39m'\u001b[39m\u001b[39m<h1>h1</h1>\u001b[39m\u001b[39m'\u001b[39m)\n",
"\u001b[0;31mAttributeError\u001b[0m: partially initialized module 'tomd' has no attribute 'Tomd' (most likely due to a circular import)"
]
}
],
"source": [
"import tomd\n",
"\n",
"tomd.convert('<h1>h1</h1>')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}

5
data_converion/tomd.py
Unescape View File

@ -1,5 +0,0 @@
import tomd
tomd.Tomd('<h1>h1</h1>').markdown
# or
tomd.convert('<h1>h1</h1>')

6
data_transfer/Add Container Inventory to Directus.ipynb
Unescape View File

@ -90,7 +90,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9.13 64-bit",
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
@ -104,14 +104,14 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
"version": "3.10.4"
},
"toc-autonumbering": false,
"toc-showcode": false,
"toc-showtags": false,
"vscode": {
"interpreter": {
"hash": "340e956ee656efd8fdfb480dc033c937d9b626f8b21073bd1b5aa2a469586ea6"
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
},
"widgets": {

2
data_transfer/Add Inventories to wiki.ipynb
Unescape View File

@ -301,7 +301,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.9"
"version": "3.10.4"
},
"toc-autonumbering": false,
"toc-showcode": false,

2
data_transfer/Add Linkace Inventory to Directus.ipynb
Unescape View File

@ -175,7 +175,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.9"
"version": "3.10.4"
},
"toc-autonumbering": false,
"toc-showcode": false,

2
data_transfer/Add traefic routers.ipynb
Unescape View File

@ -101,7 +101,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.9"
"version": "3.10.4"
},
"toc-autonumbering": false,
"toc-showcode": false,

640
documentation/python-cheatsheet.ipynb
Unescape View File

@ -0,0 +1,640 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"fatal: destination path 'python-cheatsheet' already exists and is not an empty directory.\n",
"Running `brew update --auto-update`...\n",
"\u001b[34m==>\u001b[0m \u001b[1mAuto-updated Homebrew!\u001b[0m\n",
"Updated 2 taps (go-task/tap and homebrew/core).\n",
"\u001b[34m==>\u001b[0m \u001b[1mNew Formulae\u001b[0m\n",
"aws2-wrap nmrpflash\n",
"berkeley-db@5 ocl-icd\n",
"c2rust onlykey-agent\n",
"cargo-crev open62541\n",
"cargo-depgraph openvi\n",
"chain-bench page\n",
"cpp-httplib pint\n",
"create-api pkcs11-tools\n",
"dura pocl\n",
"enex2notion prql-compiler\n",
"felinks prr\n",
"gaze pymupdf\n",
"gcc@11 sgr\n",
"gcem smap\n",
"git-delete-merged-branches snapcast\n",
"git-sync spr\n",
"go@1.18 svt-av1\n",
"gum tere\n",
"hyx tlsx\n",
"jupp treefmt\n",
"ksh93 tuntox\n",
"libbpf vile\n",
"libeatmydata vulkan-loader\n",
"lucky-commit x86_64-linux-gnu-binutils\n",
"mesheryctl\n",
"\n",
"You have \u001b[1m19\u001b[0m outdated formulae installed.\n",
"You can upgrade them with \u001b[1mbrew upgrade\u001b[0m\n",
"or list them with \u001b[1mbrew outdated\u001b[0m.\n",
"\n",
"\u001b[34m==>\u001b[0m \u001b[1mDownloading https://ghcr.io/v2/homebrew/core/fetch/manifests/0.4.5\u001b[0m\n",
"######################################################################## 100.0%\n",
"\u001b[34m==>\u001b[0m \u001b[1mDownloading https://ghcr.io/v2/homebrew/core/fetch/blobs/sha256:35d3950442bc\u001b[0m\n",
"\u001b[34m==>\u001b[0m \u001b[1mDownloading from https://pkg-containers.githubusercontent.com/ghcr1/blobs/sh\u001b[0m\n",
"######################################################################## 100.0%\n",
"\u001b[34m==>\u001b[0m \u001b[1mPouring fetch--0.4.5.x86_64_linux.bottle.tar.gz\u001b[0m\n",
"🍺 /home/linuxbrew/.linuxbrew/Cellar/fetch/0.4.5: 5 files, 8.9MB\n",
"\u001b[34m==>\u001b[0m \u001b[1mRunning `brew cleanup fetch`...\u001b[0m\n",
"Disable this behaviour by setting HOMEBREW_NO_INSTALL_CLEANUP.\n",
"Hide these hints with HOMEBREW_NO_ENV_HINTS (see `man brew`).\n"
]
}
],
"source": [
"!git clone https://github.com/wilfredinni/python-cheatsheet.git\n",
"!brew install fetch\n",
"!mkdir temp\n",
"!fetch --branch=master --source-path=/docs --repo=https://github.com/wilfredinni/python-cheatsheet temp\n",
"# https://github.com/antonbabenko/terraform-best-practices.git\n",
"%pip install markdown\n",
"%pip install python-frontmatter"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create Book if it doesn't exist\n"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\"name\":\"python-cheatsheet\",\"description\":\"python-cheatsheet\",\"created_by\":1,\"updated_by\":1,\"owned_by\":1,\"slug\":\"python-cheatsheet\",\"updated_at\":\"2022-08-10T00:14:32.000000Z\",\"created_at\":\"2022-08-10T00:14:32.000000Z\",\"id\":3}\n"
]
}
],
"source": [
"import requests\n",
"import os\n",
"\n",
"url = \"https://donavanaldrich.com/api/bookstack/books\"\n",
"\n",
"payload = {\n",
" \"name\":\n",
" \"python-cheatsheet\",\n",
" \"description\":\n",
" \"python-cheatsheet\",\n",
" \"tags\": [{\n",
" \"name\": \"Platform\",\n",
" \"value\": \"Python\"\n",
" }, {\n",
" \"name\": \"Category\",\n",
" \"value\": \"Code\"\n",
" }, {\n",
" \"name\": \"Type\",\n",
" \"value\": \"Cheatsheet\"\n",
" }]\n",
"}\n",
"headers = {\n",
" \"X-API-KEY\": os.getenv(X_API_KEY),\n",
" \"Authorization\": os.getenv(BOOKSTACK_API_TOKEN)\n",
"}\n",
"\n",
"response = requests.request(\"POST\", url, json=payload, headers=headers)\n",
"\n",
"print(response.text)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create Chapters\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### List folders to map to chapters\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['cheatsheet', 'modules', 'builtin']\n"
]
}
],
"source": [
"import os\n",
"\n",
"dir = os.listdir(\"../temp/python-cheatsheets\")\n",
"\n",
"print(dir)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"18\n",
"19\n",
"20\n"
]
}
],
"source": [
"import requests\n",
"\n",
"chapters = ['cheatsheet', 'modules', 'builtin']\n",
"\n",
"url = \"https://donavanaldrich.com/api/bookstack/chapters\"\n",
"\n",
"for chapter_name in chapters:\n",
"\n",
" payload = {\n",
" \"book_id\": 2,\n",
" \"name\": chapter_name,\n",
" \"description\": chapter_name,\n",
" \"tags\": [{\n",
" \"name\": \"chapter\",\n",
" \"value\": chapter_name\n",
" }]\n",
" }\n",
" headers = {\n",
" \"X-API-KEY\": os.getenv(X_API_KEY),\n",
" \"Authorization\": os.getenv(BOOKSTACK_API_TOKEN)\n",
" }\n",
"\n",
" response = requests.request(\"POST\", url, json=payload, headers=headers)\n",
" response_json = response.json()\n",
" chapter_id = response_json['id']\n",
" print(chapter_id)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Add Pages\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'X_API_KEY' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb Cell 9\u001b[0m in \u001b[0;36m<cell line: 10>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=5'>6</a>\u001b[0m chapters \u001b[39m=\u001b[39m [\u001b[39m'\u001b[39m\u001b[39mcheatsheet\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mmodules\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mbuiltin\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=6'>7</a>\u001b[0m \u001b[39m# The Python interpreter has a number of functions and types built into it that are always available.\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39m# Standard Library\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=8'>9</a>\u001b[0m headers \u001b[39m=\u001b[39m {\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mX-API-KEY\u001b[39m\u001b[39m\"\u001b[39m: os\u001b[39m.\u001b[39mgetenv(X_API_KEY),\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mAuthorization\u001b[39m\u001b[39m\"\u001b[39m: os\u001b[39m.\u001b[39mgetenv(BOOKSTACK_API_TOKEN)\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=11'>12</a>\u001b[0m }\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=13'>14</a>\u001b[0m url \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mhttps://donavanaldrich.com/api/bookstack/books\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=15'>16</a>\u001b[0m payload \u001b[39m=\u001b[39m {\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=16'>17</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mname\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=17'>18</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mpython-cheatsheet\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=29'>30</a>\u001b[0m }]\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=30'>31</a>\u001b[0m }\n",
"\u001b[0;31mNameError\u001b[0m: name 'X_API_KEY' is not defined"
]
}
],
"source": [
"from pathlib import Path\n",
"import requests\n",
"import frontmatter\n",
"\n",
"chapters = ['cheatsheet', 'modules', 'builtin']\n",
"\n",
"headers = {\n",
" \"X-API-KEY\": os.getenv(X_API_KEY),\n",
" \"Authorization\": os.getenv(BOOKSTACK_API_TOKEN)\n",
"}\n",
"\n",
"url = \"https://donavanaldrich.com/api/bookstack/books\"\n",
"\n",
"payload = {\n",
" \"name\":\n",
" \"python-cheatsheet\",\n",
" \"description\":\n",
" \"python-cheatsheet\",\n",
" \"tags\": [{\n",
" \"name\": \"Platform\",\n",
" \"value\": \"Python\"\n",
" }, {\n",
" \"name\": \"Category\",\n",
" \"value\": \"Code\"\n",
" }, {\n",
" \"name\": \"Type\",\n",
" \"value\": \"Cheatsheet\"\n",
" }]\n",
"}\n",
"\n",
"response = requests.request(\"POST\", url, json=payload, headers=headers)\n",
"response_json = response.json()\n",
"book_id = response_json['id']\n",
"\n",
"# book_id = xs\n",
"#\n",
"for chapter in chapters:\n",
" payload = {\n",
" \"book_id\": book_id,\n",
" \"name\": chapter,\n",
" \"description\": chapter,\n",
" \"tags\": [{\n",
" \"name\": \"chapter\",\n",
" \"value\": chapter\n",
" }]\n",
" }\n",
" url = \"https://donavanaldrich.com/api/bookstack/chapters\"\n",
" response = requests.request(\"POST\", url, json=payload, headers=headers)\n",
" response_json = response.json()\n",
" chapter_id = response_json['id']\n",
" print(chapter_id)\n",
"\n",
" for page in Path('../temp/python-cheatsheets/' + chapter).iterdir():\n",
" url = \"https://donavanaldrich.com/api/bookstack/pages\"\n",
" with open(page, 'r') as f:\n",
" metadata, content = frontmatter.parse(f.read())\n",
" try:\n",
" page_name = metadata['title']\n",
" except:\n",
" page_name = str(page)\n",
" payload = {\n",
" \"book_id\": book_id,\n",
" \"chapter_id\": chapter_id,\n",
" \"name\": page_name,\n",
" \"markdown\": content,\n",
" \"tags\": [{\n",
" \"name\": \"chapter\",\n",
" \"value\": chapter\n",
" }]\n",
" }\n",
" response = requests.request(\"POST\",\n",
" url,\n",
" json=payload,\n",
" headers=headers)\n",
"\n",
" # print(response.text)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Development/Sandboxing\n"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"# Python Debugging\n",
"\n",
"<base-disclaimer>\n",
" <base-disclaimer-title>\n",
" <a target=\"_blank\" href=\"https://en.wikipedia.org/wiki/Debugging\">Finding and resolving bugs</a>\n",
" </base-disclaimer-title>\n",
" <base-disclaimer-content>\n",
" In computer programming and software development, debugging is the process of finding and resolving bugs (defects or problems that prevent correct operation) within computer programs, software, or systems.\n",
" </base-disclaimer-content>\n",
"</base-disclaimer>\n",
"\n",
"## Raising Exceptions\n",
"\n",
"Exceptions are raised with a raise statement. In code, a raise statement consists of the following:\n",
"\n",
"- The `raise` keyword\n",
"- A call to the `Exception()` function\n",
"- A string with a helpful error message passed to the `Exception()` function\n",
"\n",
"```python\n",
">>> raise Exception('This is the error message.')\n",
"# Traceback (most recent call last):\n",
"# File \"<pyshell#191>\", line 1, in <module>\n",
"# raise Exception('This is the error message.')\n",
"# Exception: This is the error message.\n",
"```\n",
"\n",
"Typically, it’s the code that calls the function, not the function itself, that knows how to handle an exception. So, you will commonly see a raise statement inside a function and the `try` and `except` statements in the code calling the function.\n",
"\n",
"```python\n",
">>> def box_print(symbol, width, height):\n",
"... if len(symbol) != 1:\n",
"... raise Exception('Symbol must be a single character string.')\n",
"... if width <= 2:\n",
"... raise Exception('Width must be greater than 2.')\n",
"... if height <= 2:\n",
"... raise Exception('Height must be greater than 2.')\n",
"... print(symbol * width)\n",
"... for i in range(height - 2):\n",
"... print(symbol + (' ' * (width - 2)) + symbol)\n",
"... print(symbol * width)\n",
"...\n",
">>> for sym, w, h in (('*', 4, 4), ('O', 20, 5), ('x', 1, 3), ('ZZ', 3, 3)):\n",
"... try:\n",
"... box_print(sym, w, h)\n",
"... except Exception as err:\n",
"... print('An exception happened: ' + str(err))\n",
"...\n",
"# ****\n",
"# * *\n",
"# * *\n",
"# ****\n",
"# OOOOOOOOOOOOOOOOOOOO\n",
"# O O\n",
"# O O\n",
"# O O\n",
"# OOOOOOOOOOOOOOOOOOOO\n",
"# An exception happened: Width must be greater than 2.\n",
"# An exception happened: Symbol must be a single character string.\n",
"```\n",
"\n",
"Read more about [Exception Handling](/cheatsheet/exception-handling).\n",
"\n",
"## Getting the Traceback as a string\n",
"\n",
"The `traceback` is displayed by Python whenever a raised exception goes unhandled. But can also obtain it as a string by calling traceback.format_exc(). This function is useful if you want the information from an exception’s traceback but also want an except statement to gracefully handle the exception. You will need to import Python’s traceback module before calling this function.\n",
"\n",
"```python\n",
">>> import traceback\n",
"\n",
">>> try:\n",
"... raise Exception('This is the error message.')\n",
">>> except:\n",
"... with open('errorInfo.txt', 'w') as error_file:\n",
"... error_file.write(traceback.format_exc())\n",
"... print('The traceback info was written to errorInfo.txt.')\n",
"...\n",
"# 116\n",
"# The traceback info was written to errorInfo.txt.\n",
"```\n",
"\n",
"The 116 is the return value from the `write()` method, since 116 characters were written to the file. The `traceback` text was written to errorInfo.txt.\n",
"\n",
" Traceback (most recent call last):\n",
" File \"<pyshell#28>\", line 2, in <module>\n",
" Exception: This is the error message.\n",
"\n",
"## Assertions\n",
"\n",
"An assertion is a sanity check to make sure your code isn’t doing something obviously wrong. These sanity checks are performed by `assert` statements. If the sanity check fails, then an `AssertionError` exception is raised. In code, an `assert` statement consists of the following:\n",
"\n",
"- The `assert` keyword\n",
"- A condition (that is, an expression that evaluates to `True` or `False`)\n",
"- A comma\n",
"- A `string` to display when the condition is `False`\n",
"\n",
"```python\n",
">>> pod_bay_door_status = 'open'\n",
">>> assert pod_bay_door_status == 'open', 'The pod bay doors need to be \"open\".'\n",
"\n",
">>> pod_bay_door_status = 'I\\'m sorry, Dave. I\\'m afraid I can\\'t do that.'\n",
">>> assert pod_bay_door_status == 'open', 'The pod bay doors need to be \"open\".'\n",
"# Traceback (most recent call last):\n",
"# File \"<pyshell#10>\", line 1, in <module>\n",
"# assert pod_bay_door_status == 'open', 'The pod bay doors need to be \"open\".'\n",
"# AssertionError: The pod bay doors need to be \"open\".\n",
"```\n",
"\n",
"In plain English, an assert statement says, “I assert that this condition holds true, and if not, there is a bug somewhere in the program.” Unlike exceptions, your code should not handle assert statements with try and except; if an assert fails, your program should crash. By failing fast like this, you shorten the time between the original cause of the bug and when you first notice the bug. This will reduce the amount of code you will have to check before finding the code that’s causing the bug.\n",
"\n",
"### Disabling Assertions\n",
"\n",
"Assertions can be disabled by passing the `-O` option when running Python.\n",
"\n",
"## Logging\n",
"\n",
"To enable the `logging` module to display log messages on your screen as your program runs, copy the following to the top of your program:\n",
"\n",
"```python\n",
">>> import logging\n",
">>> logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s - %(levelname)s- %(message)s')\n",
"```\n",
"\n",
"Say you wrote a function to calculate the factorial of a number. In mathematics, factorial 4 is 1 × 2 × 3 × 4, or 24. Factorial 7 is 1 × 2 × 3 × 4 × 5 × 6 × 7, or 5,040. Open a new file editor window and enter the following code. It has a bug in it, but you will also enter several log messages to help yourself figure out what is going wrong. Save the program as factorialLog.py.\n",
"\n",
"```python\n",
">>> import logging\n",
">>> logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s - %(levelname)s- %(message)s')\n",
">>> logging.debug('Start of program')\n",
"\n",
">>> def factorial(n):\n",
"... logging.debug('Start of factorial(%s)' % (n))\n",
"... total = 1\n",
"... for i in range(1, n + 1):\n",
"... total *= i\n",
"... logging.debug('i is ' + str(i) + ', total is ' + str(total))\n",
"... logging.debug('End of factorial(%s)' % (n))\n",
"... return total\n",
"...\n",
">>> print(factorial(5))\n",
">>> logging.debug('End of program')\n",
"# 2015-05-23 16:20:12,664 - DEBUG - Start of program\n",
"# 2015-05-23 16:20:12,664 - DEBUG - Start of factorial(5)\n",
"# 2015-05-23 16:20:12,665 - DEBUG - i is 0, total is 0\n",
"# 2015-05-23 16:20:12,668 - DEBUG - i is 1, total is 0\n",
"# 2015-05-23 16:20:12,670 - DEBUG - i is 2, total is 0\n",
"# 2015-05-23 16:20:12,673 - DEBUG - i is 3, total is 0\n",
"# 2015-05-23 16:20:12,675 - DEBUG - i is 4, total is 0\n",
"# 2015-05-23 16:20:12,678 - DEBUG - i is 5, total is 0\n",
"# 2015-05-23 16:20:12,680 - DEBUG - End of factorial(5)\n",
"# 0\n",
"# 2015-05-23 16:20:12,684 - DEBUG - End of program\n",
"```\n",
"\n",
"## Logging Levels\n",
"\n",
"Logging levels provide a way to categorize your log messages by importance. There are five logging levels, described in Table 10-1 from least to most important. Messages can be logged at each level using a different logging function.\n",
"\n",
"| Level | Logging Function | Description |\n",
"| ---------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------ |\n",
"| `DEBUG` | `logging.debug()` | The lowest level. Used for small details. Usually you care about these messages only when diagnosing problems. |\n",
"| `INFO` | `logging.info()` | Used to record information on general events in your program or confirm that things are working at their point in the program. |\n",
"| `WARNING` | `logging.warning()` | Used to indicate a potential problem that doesn’t prevent the program from working but might do so in the future. |\n",
"| `ERROR` | `logging.error()` | Used to record an error that caused the program to fail to do something. |\n",
"| `CRITICAL` | `logging.critical()` | The highest level. Used to indicate a fatal error that has caused or is about to cause the program to stop running entirely. |\n",
"\n",
"## Disabling Logging\n",
"\n",
"After you’ve debugged your program, you probably don’t want all these log messages cluttering the screen. The logging.disable() function disables these so that you don’t have to go into your program and remove all the logging calls by hand.\n",
"\n",
"```python\n",
">>> import logging\n",
"\n",
">>> logging.basicConfig(level=logging.INFO, format=' %(asctime)s -%(levelname)s - %(message)s')\n",
">>> logging.critical('Critical error! Critical error!')\n",
"# 2015-05-22 11:10:48,054 - CRITICAL - Critical error! Critical error!\n",
"\n",
">>> logging.disable(logging.CRITICAL)\n",
">>> logging.critical('Critical error! Critical error!')\n",
">>> logging.error('Error! Error!')\n",
"```\n",
"\n",
"## Logging to a File\n",
"\n",
"Instead of displaying the log messages to the screen, you can write them to a text file. The `logging.basicConfig()` function takes a filename keyword argument, like so:\n",
"\n",
"```python\n",
">>> import logging\n",
">>> logging.basicConfig(filename='myProgramLog.txt', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')\n",
"```\n"
]
}
],
"source": [
"# for page in Path('../temp/python-cheatsheets/' + chapter).iterdir():\n",
"page = '../temp/python-cheatsheets/cheatsheet/debugging.md'\n",
"url = \"https://donavanaldrich.com/api/bookstack/pages\"\n",
"\n",
"with open(page, 'r') as f:\n",
" # text = f.read()\n",
" # post = frontmatter.load(page)\n",
" metadata, content = frontmatter.parse(f.read())\n",
"print(content)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Reference\n",
"\n",
"### Payloads\n",
"\n",
"#### book\n",
"\n",
"```json\n",
"{\n",
" \"name\": \"python-cheatsheet\",\n",
" \"description\": \"python-cheatsheet\",\n",
" \"tags\": [\n",
" { \"name\": \"Platform\", \"value\": \"Python\" },\n",
" { \"name\": \"Category\", \"value\": \"Code\" },\n",
" { \"name\": \"Type\", \"value\": \"Cheatsheet\" }\n",
" ]\n",
"}\n",
"```\n",
"\n",
"#### Chapter\n",
"\n",
"```json\n",
"{\n",
" \"book_id\": 1,\n",
" \"name\": \"My fantastic new chapter\",\n",
" \"description\": \"This is a great new chapter that I've created via the API\",\n",
" \"tags\": [\n",
" { \"name\": \"Category\", \"value\": \"Top Content\" },\n",
" { \"name\": \"Rating\", \"value\": \"Highest\" }\n",
" ]\n",
"}\n",
"```\n",
"\n",
"#### Page\n",
"\n",
"```json\n",
"{\n",
" \"book_id\": 1,\n",
" \"name\": \"My API Page\",\n",
" \"markdown\": \"<p>my new API page</p>\",\n",
" \"tags\": [\n",
" { \"name\": \"Category\", \"value\": \"Not Bad Content\" },\n",
" { \"name\": \"Rating\", \"value\": \"Average\" }\n",
" ]\n",
"}\n",
"```\n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "module 'frontmatter' has no attribute 'load'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb Cell 19\u001b[0m in \u001b[0;36m<cell line: 13>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=5'>6</a>\u001b[0m file_parts \u001b[39m=\u001b[39m frontmatter\u001b[39m.\u001b[39mload(filepath)\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39mreturn\u001b[39;00m {\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=8'>9</a>\u001b[0m \u001b[39m# 'html': markdown_parser.convert(file_parts.content),\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39m'\u001b[39m\u001b[39mmetadata\u001b[39m\u001b[39m'\u001b[39m: file_parts\u001b[39m.\u001b[39mmetadata\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m }\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=12'>13</a>\u001b[0m parse_markdown(\u001b[39m'\u001b[39;49m\u001b[39m../temp/python-cheatsheets/cheatsheet/debugging.md\u001b[39;49m\u001b[39m'\u001b[39;49m)\n",
"\u001b[1;32m/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb Cell 19\u001b[0m in \u001b[0;36mparse_markdown\u001b[0;34m(filepath)\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3'>4</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mparse_markdown\u001b[39m(filepath):\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=4'>5</a>\u001b[0m \u001b[39m# markdown_parser = markdown.Markdown(extensions=markdown_extensions)\u001b[39;00m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=5'>6</a>\u001b[0m file_parts \u001b[39m=\u001b[39m frontmatter\u001b[39m.\u001b[39;49mload(filepath)\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39mreturn\u001b[39;00m {\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=8'>9</a>\u001b[0m \u001b[39m# 'html': markdown_parser.convert(file_parts.content),\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39m'\u001b[39m\u001b[39mmetadata\u001b[39m\u001b[39m'\u001b[39m: file_parts\u001b[39m.\u001b[39mmetadata\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/documentation/python-cheatsheet.ipynb#X23sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m }\n",
"\u001b[0;31mAttributeError\u001b[0m: module 'frontmatter' has no attribute 'load'"
]
}
],
"source": [
"import frontmatter # https://pypi.org/project/python-frontmatter/\n",
"import markdown\n",
"\n",
"\n",
"def parse_markdown(filepath):\n",
" # markdown_parser = markdown.Markdown(extensions=markdown_extensions)\n",
" file_parts = frontmatter.load(filepath)\n",
"\n",
" return {\n",
" # 'html': markdown_parser.convert(file_parts.content),\n",
" 'metadata': file_parts.metadata\n",
" }\n",
"\n",
"\n",
"parse_markdown('../temp/python-cheatsheets/cheatsheet/debugging.md')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.8.9 64-bit",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}

2485
documentation/terraform-cheatsheet.ipynb
View File

File diff suppressed because one or more lines are too long

119
documentation/terraform-docs.ipynb
Unescape View File

@ -0,0 +1,119 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"!git clone https://github.com/hashicorp/terraform.git\n",
"\n",
"https://github.com/antonbabenko/terraform-best-practices.git\n",
"\n",
"https://github.com/hashicorp/terraform-guides.git"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Running `brew update --auto-update`...\n",
"\u001b[34m==>\u001b[0m \u001b[1mAuto-updated Homebrew!\u001b[0m\n",
"Updated 1 tap (homebrew/core).\n",
"\n",
"You have \u001b[1m19\u001b[0m outdated formulae installed.\n",
"You can upgrade them with \u001b[1mbrew upgrade\u001b[0m\n",
"or list them with \u001b[1mbrew outdated\u001b[0m.\n",
"\n",
"\u001b[33mWarning:\u001b[0m fetch 0.4.5 is already installed and up-to-date.\n",
"To reinstall 0.4.5, run:\n",
" brew reinstall fetch\n",
"mkdir: cannot create directory ‘../temp’: File exists\n",
"mkdir: cannot create directory ‘../temp/terraform/docs’: No such file or directory\n",
"mkdir: cannot create directory ‘../temp/terraform/data’: No such file or directory\n",
"[fetch] \u001b[36mINFO\u001b[0m[2022-08-11T01:35:41Z] Downloading latest commit from branch \"master\" of https://github.com/hashicorp/terraform ... \n",
"[fetch] \u001b[36mINFO\u001b[0m[2022-08-11T01:35:42Z] Extracting files from <repo>/website/docs to ../temp/terraform/docs ... \n",
"[fetch] \u001b[36mINFO\u001b[0m[2022-08-11T01:35:42Z] 329 files extracted \n",
"[fetch] \u001b[36mINFO\u001b[0m[2022-08-11T01:35:42Z] Download and file extraction complete. \n",
"[fetch] \u001b[36mINFO\u001b[0m[2022-08-11T01:35:42Z] Downloading latest commit from branch \"master\" of https://github.com/hashicorp/terraform ... \n",
"[fetch] \u001b[36mINFO\u001b[0m[2022-08-11T01:35:43Z] Extracting files from <repo>/website/data to ../temp/terraform/data ... \n",
"[fetch] \u001b[36mINFO\u001b[0m[2022-08-11T01:35:43Z] 5 files extracted \n",
"[fetch] \u001b[36mINFO\u001b[0m[2022-08-11T01:35:43Z] Download and file extraction complete. \n",
"Defaulting to user installation because normal site-packages is not writeable\n",
"Requirement already satisfied: markdown in /home/donaldrich/.local/lib/python3.10/site-packages (3.4.1)\n",
"Note: you may need to restart the kernel to use updated packages.\n",
"Defaulting to user installation because normal site-packages is not writeable\n",
"Requirement already satisfied: python-frontmatter in /home/donaldrich/.local/lib/python3.10/site-packages (1.0.0)\n",
"Requirement already satisfied: PyYAML in /home/donaldrich/.local/lib/python3.10/site-packages (from python-frontmatter) (5.1)\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
}
],
"source": [
"!brew install fetch\n",
"\n",
"!mkdir ../temp\n",
"!mkdir ../temp/terraform/docs\n",
"!mkdir ../temp/terraform/data\n",
"\n",
"!fetch --branch=master --source-path=/website/docs --repo=https://github.com/hashicorp/terraform ../temp/terraform/docs\n",
"!fetch --branch=master --source-path=/website/data --repo=https://github.com/hashicorp/terraform ../temp/terraform/data\n",
"\n",
"\n",
"%pip install markdown\n",
"%pip install python-frontmatter"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['data', 'docs']\n"
]
}
],
"source": [
"import os\n",
"\n",
"dir = os.listdir(\"../temp/terraform\")\n",
"\n",
"print(dir)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}

18
documentation/weasyprint.ipynb
Unescape View File

@ -58,7 +58,7 @@
}
],
"source": [
"!pip install weasyprint"
"%pip install weasyprint"
]
},
{
@ -88,21 +88,11 @@
"\n",
"HTML(\"http://weasyprint.org/\").write_pdf(\"/User/donaldrich/Desktop/weasyprint.pdf\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b66cdca8-02c4-4761-84b1-a0a91991fb0a",
"metadata": {},
"outputs": [],
"source": [
"https://selenium-python.readthedocs.io/_/downloads/en/latest/pdf/"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9.13 64-bit",
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
@ -116,11 +106,11 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
"version": "3.10.4"
},
"vscode": {
"interpreter": {
"hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
},
"widgets": {

287
http/furl/furl.ipynb
Unescape View File

@ -0,0 +1,287 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# furl\n",
"\n",
"[Github](https://github.com/gruns/furl)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
" Dload Upload Total Spent Left Speed\n",
" 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 185.199.109.133:443...\n",
"* Connected to raw.githubusercontent.com (185.199.109.133) port 443 (#0)\n",
"* ALPN, offering h2\n",
"* ALPN, offering http/1.1\n",
"* CAfile: /etc/ssl/certs/ca-certificates.crt\n",
"* CApath: /etc/ssl/certs\n",
"* TLSv1.0 (OUT), TLS header, Certificate Status (22):\n",
"} [5 bytes data]\n",
"* TLSv1.3 (OUT), TLS handshake, Client hello (1):\n",
"} [512 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Certificate Status (22):\n",
"{ [5 bytes data]\n",
"* TLSv1.3 (IN), TLS handshake, Server hello (2):\n",
"{ [122 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Finished (20):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):\n",
"{ [19 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.3 (IN), TLS handshake, Certificate (11):\n",
"{ [3051 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.3 (IN), TLS handshake, CERT verify (15):\n",
"{ [264 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.3 (IN), TLS handshake, Finished (20):\n",
"{ [52 bytes data]\n",
"* TLSv1.2 (OUT), TLS header, Finished (20):\n",
"} [5 bytes data]\n",
"* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):\n",
"} [1 bytes data]\n",
"* TLSv1.2 (OUT), TLS header, Supplemental data (23):\n",
"} [5 bytes data]\n",
"* TLSv1.3 (OUT), TLS handshake, Finished (20):\n",
"} [52 bytes data]\n",
"* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384\n",
"* ALPN, server accepted to use h2\n",
"* Server certificate:\n",
"* subject: C=US; ST=California; L=San Francisco; O=GitHub, Inc.; CN=*.github.io\n",
"* start date: Mar 18 00:00:00 2022 GMT\n",
"* expire date: Mar 21 23:59:59 2023 GMT\n",
"* subjectAltName: host \"raw.githubusercontent.com\" matched cert's \"*.githubusercontent.com\"\n",
"* issuer: C=US; O=DigiCert Inc; CN=DigiCert TLS RSA SHA256 2020 CA1\n",
"* SSL certificate verify ok.\n",
"* Using HTTP2, server supports multiplexing\n",
"* Connection state changed (HTTP/2 confirmed)\n",
"* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0\n",
"* TLSv1.2 (OUT), TLS header, Supplemental data (23):\n",
"} [5 bytes data]\n",
"* TLSv1.2 (OUT), TLS header, Supplemental data (23):\n",
"} [5 bytes data]\n",
"* TLSv1.2 (OUT), TLS header, Supplemental data (23):\n",
"} [5 bytes data]\n",
"* Using Stream ID: 1 (easy handle 0x559fdfac8e80)\n",
"* TLSv1.2 (OUT), TLS header, Supplemental data (23):\n",
"} [5 bytes data]\n",
"> GET /gruns/furl/master/README.md HTTP/2\n",
"> Host: raw.githubusercontent.com\n",
"> user-agent: curl/7.81.0\n",
"> accept: */*\n",
"> \n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):\n",
"{ [193 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (OUT), TLS header, Supplemental data (23):\n",
"} [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"< HTTP/2 200 \n",
"< cache-control: max-age=300\n",
"< content-security-policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\n",
"< content-type: text/plain; charset=utf-8\n",
"< etag: \"a422674a44f080e982c2757a03c7a6085f6de3a85731442a2f14785510ef12e2\"\n",
"< strict-transport-security: max-age=31536000\n",
"< x-content-type-options: nosniff\n",
"< x-frame-options: deny\n",
"< x-xss-protection: 1; mode=block\n",
"< x-github-request-id: 5F04:63F8:20E44:3B90F:62F2032B\n",
"< accept-ranges: bytes\n",
"< date: Tue, 09 Aug 2022 06:48:11 GMT\n",
"< via: 1.1 varnish\n",
"< x-served-by: cache-iah17220-IAH\n",
"< x-cache: MISS\n",
"< x-cache-hits: 0\n",
"< x-timer: S1660027691.995129,VS0,VE126\n",
"< vary: Authorization,Accept-Encoding,Origin\n",
"< access-control-allow-origin: *\n",
"< x-fastly-request-id: 3ae1114575f72aef60b2af9ef7e0d60266073cbb\n",
"< expires: Tue, 09 Aug 2022 06:53:11 GMT\n",
"< source-age: 0\n",
"< content-length: 23922\n",
"< \n",
"{ [885 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"* TLSv1.2 (IN), TLS header, Supplemental data (23):\n",
"{ [5 bytes data]\n",
"100 23922 100 23922 0 0 76972 0 --:--:-- --:--:-- --:--:-- 77167\n",
"* Connection #0 to host raw.githubusercontent.com left intact\n"
]
}
],
"source": [
"# Retrieve Decumentation\n",
"\n",
"!curl -fLvo readme.md https://raw.githubusercontent.com/gruns/furl/master/README.md"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Defaulting to user installation because normal site-packages is not writeable\n",
"Collecting furl\n",
" Downloading furl-2.1.3-py2.py3-none-any.whl (20 kB)\n",
"Requirement already satisfied: six>=1.8.0 in /usr/lib/python3/dist-packages (from furl) (1.16.0)\n",
"Collecting orderedmultidict>=1.0.1\n",
" Downloading orderedmultidict-1.0.1-py2.py3-none-any.whl (11 kB)\n",
"Installing collected packages: orderedmultidict, furl\n",
"Successfully installed furl-2.1.3 orderedmultidict-1.0.1\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
}
],
"source": [
"%pip install furl"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Function: URL Deconstructor\n",
"\n",
"Takes a URL, chops it up into all the little pieces, returns as a JSON dictionary"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'fragment': {'encoded': 'scheme-username-password-host-port-network-location-and-origin',\n",
" 'path': {'encoded': 'scheme-username-password-host-port-network-location-and-origin',\n",
" 'isabsolute': False,\n",
" 'isdir': False,\n",
" 'isfile': True,\n",
" 'segments': ['scheme-username-password-host-port-network-location-and-origin']},\n",
" 'query': {'encoded': '', 'params': []},\n",
" 'separator': True},\n",
" 'host': 'github.com',\n",
" 'host_encoded': 'github.com',\n",
" 'netloc': 'github.com',\n",
" 'origin': 'https://github.com',\n",
" 'password': None,\n",
" 'path': {'encoded': '/gruns/furl',\n",
" 'isabsolute': True,\n",
" 'isdir': False,\n",
" 'isfile': True,\n",
" 'segments': ['gruns', 'furl']},\n",
" 'port': 443,\n",
" 'query': {'encoded': '', 'params': []},\n",
" 'scheme': 'https',\n",
" 'url': 'https://github.com/gruns/furl#scheme-username-password-host-port-network-location-and-origin',\n",
" 'username': None}\n"
]
}
],
"source": [
"from furl import furl\n",
"import pprint\n",
"\n",
"url = 'https://github.com/gruns/furl#scheme-username-password-host-port-network-location-and-origin'\n",
"\n",
"f = furl(url)\n",
"\n",
"result = f.asdict()\n",
"\n",
"pprint.pprint(result)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.8.10 64-bit",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}

784
http/furl/readme.md
Unescape View File

@ -0,0 +1,784 @@
<h1 align="center">
<img src="logo.svg" width="360px" height="123px" alt="furl">
</h1>
<p align="center">
<a href="https://pypi.python.org/pypi/furl"><img src="https://badge.fury.io/py/furl.svg"></a>
<a href="https://travis-ci.org/gruns/furl"><img src="https://img.shields.io/travis/gruns/furl.svg"></a>
<a href="http://unlicense.org/"><img src="https://img.shields.io/pypi/l/furl.svg"></a>
<a href="https://pypi.python.org/pypi/furl"><img src="https://img.shields.io/pypi/pyversions/furl.svg"></a>
</p>
## furl is a small Python library that makes parsing and<br>manipulating URLs easy.
Python's standard [urllib](https://docs.python.org/3/library/urllib.html) and
[urlparse](https://docs.python.org/3/library/urllib.parse.html) modules provide a
number of URL\
related functions, but using these functions to perform common
URL\
operations proves tedious. Furl makes parsing and manipulating URLs\
easy.
Furl is well tested, [Unlicensed](http://unlicense.org/) in the public domain,
and supports\
Python 2, Python 3, PyPy2, and PyPy3.
Code time: Paths and query arguments are easy. Really easy.
```python
>>> from furl import furl
>>> f = furl('http://www.google.com/?one=1&two=2')
>>> f /= 'path'
>>> del f.args['one']
>>> f.args['three'] = '3'
>>> f.url
'http://www.google.com/path?two=2&three=3'
```
Or use furl's inline modification methods.
```python
>>> furl('http://www.google.com/?one=1').add({'two':'2'}).url
'http://www.google.com/?one=1&two=2'
>>> furl('http://www.google.com/?one=1&two=2').set({'three':'3'}).url
'http://www.google.com/?three=3'
>>> furl('http://www.google.com/?one=1&two=2').remove(['one']).url
'http://www.google.com/?two=2'
```
Encoding is handled for you. Unicode, too.
```python
>>> f = furl('http://www.google.com/')
>>> f.path = 'some encoding here'
>>> f.args['and some encoding'] = 'here, too'
>>> f.url
'http://www.google.com/some%20encoding%20here?and+some+encoding=here,+too'
>>> f.set(host=u'ドメイン.テスト', path=u'джк', query=u'☃=☺')
>>> f.url
'http://xn--eckwd4c7c.xn--zckzah/%D0%B4%D0%B6%D0%BA?%E2%98%83=%E2%98%BA'
```
Fragments also have a path and a query.
```python
>>> f = furl('http://www.google.com/')
>>> f.fragment.path.segments = ['two', 'directories']
>>> f.fragment.args = {'one': 'argument'}
>>> f.url
'http://www.google.com/#two/directories?one=argument'
```
## Installation
Installing furl with pip is easy.
```
$ pip install furl
```
## API
* [Basics](#basics)
* [Scheme, Username, Password, Host, Port, Network Location, and Origin](#scheme-username-password-host-port-network-location-and-origin)
* [Path](#path)
* [Manipulation](#manipulation)
* [Query](#query)
* [Manipulation](#manipulation-1)
* [Parameters](#parameters)
* [Fragment](#fragment)
* [Encoding](#encoding)
* [Inline manipulation](#inline-manipulation)
* [Miscellaneous](#miscellaneous)
### Basics
furl objects let you access and modify the various components of a URL.
```
scheme://username:password@host:port/path?query#fragment
```
* __scheme__ is the scheme string (all lowercase) or None. None means no
scheme. An empty string means a protocol relative URL, like
`//www.google.com`.
* __username__ is the username string for authentication.
* __password__ is the password string for authentication with __username__.
* __host__ is the domain name, IPv4, or IPv6 address as a string. Domain names
are all lowercase.
* __port__ is an integer or None. A value of None means no port specified and
the default port for the given __scheme__ should be inferred, if possible
(e.g. port 80 for the scheme `http`).
* __path__ is a Path object comprised of path segments.
* __query__ is a Query object comprised of key:value query arguments.
* __fragment__ is a Fragment object comprised of a Path object and Query object
separated by an optional `?` separator.
### Scheme, Username, Password, Host, Port, Network Location, and Origin
__scheme__, __username__, __password__, and __host__ are strings or
None. __port__ is an integer or None.
```python
>>> f = furl('http://user:pass@www.google.com:99/')
>>> f.scheme, f.username, f.password, f.host, f.port
('http', 'user', 'pass', 'www.google.com', 99)
```
furl infers the default port for common schemes.
```python
>>> f = furl('https://secure.google.com/')
>>> f.port
443
>>> f = furl('unknown://www.google.com/')
>>> print(f.port)
None
```
__netloc__ is the string combination of __username__, __password__, __host__,
and __port__, not including __port__ if it's None or the default port for the
provided __scheme__.
```python
>>> furl('http://www.google.com/').netloc
'www.google.com'
>>> furl('http://www.google.com:99/').netloc
'www.google.com:99'
>>> furl('http://user:pass@www.google.com:99/').netloc
'user:pass@www.google.com:99'
```
__origin__ is the string combination of __scheme__, __host__, and __port__, not
including __port__ if it's None or the default port for the provided __scheme__.
```python
>>> furl('http://www.google.com/').origin
'http://www.google.com'
>>> furl('http://www.google.com:99/').origin
'http://www.google.com:99'
```
### Path
URL paths in furl are Path objects that have __segments__, a list of zero or
more path segments that can be manipulated directly. Path segments in
__segments__ are percent-decoded and all interaction with __segments__ should
take place with percent-decoded strings.
```python
>>> f = furl('http://www.google.com/a/large%20ish/path')
>>> f.path
Path('/a/large ish/path')
>>> f.path.segments
['a', 'large ish', 'path']
>>> str(f.path)
'/a/large%20ish/path'
```
#### Manipulation
```python
>>> f.path.segments = ['a', 'new', 'path', '']
>>> str(f.path)
'/a/new/path/'
>>> f.path = 'o/hi/there/with%20some%20encoding/'
>>> f.path.segments
['o', 'hi', 'there', 'with some encoding', '']
>>> str(f.path)
'/o/hi/there/with%20some%20encoding/'
>>> f.url
'http://www.google.com/o/hi/there/with%20some%20encoding/'
>>> f.path.segments = ['segments', 'are', 'maintained', 'decoded', '^`<>[]"#/?']
>>> str(f.path)
'/segments/are/maintained/decoded/%5E%60%3C%3E%5B%5D%22%23%2F%3F'
```
A path that starts with `/` is considered absolute, and a Path can be absolute
or not as specified (or set) by the boolean attribute __isabsolute__. URL Paths
have a special restriction: they must be absolute if a __netloc__ (username,
password, host, and/or port) is present. This restriction exists because a URL
path must start with `/` to separate itself from the __netloc__, if
present. Fragment Paths have no such limitation and __isabsolute__ and can be
True or False without restriction.
Here's a URL Path example that illustrates how __isabsolute__ becomes True and
read-only in the presence of a __netloc__.
```python
>>> f = furl('/url/path')
>>> f.path.isabsolute
True
>>> f.path.isabsolute = False
>>> f.url
'url/path'
>>> f.host = 'blaps.ru'
>>> f.url
'blaps.ru/url/path'
>>> f.path.isabsolute
True
>>> f.path.isabsolute = False
Traceback (most recent call last):
...
AttributeError: Path.isabsolute is True and read-only for URLs with a netloc (a username, password, host, and/or port). URL paths must be absolute if a netloc exists.
>>> f.url
'blaps.ru/url/path'
```
Conversely, the __isabsolute__ attribute of Fragment Paths isn't bound by the
same read-only restriction. URL fragments are always prefixed by a `#` character
and don't need to be separated from the __netloc__.
```python
>>> f = furl('http://www.google.com/#/absolute/fragment/path/')
>>> f.fragment.path.isabsolute
True
>>> f.fragment.path.isabsolute = False
>>> f.url
'http://www.google.com/#absolute/fragment/path/'
>>> f.fragment.path.isabsolute = True
>>> f.url
'http://www.google.com/#/absolute/fragment/path/'
```
A path that ends with `/` is considered a directory, and otherwise considered a
file. The Path attribute __isdir__ returns True if the path is a directory,
False otherwise. Conversely, the attribute __isfile__ returns True if the path
is a file, False otherwise.
```python
>>> f = furl('http://www.google.com/a/directory/')
>>> f.path.isdir
True
>>> f.path.isfile
False
>>> f = furl('http://www.google.com/a/file')
>>> f.path.isdir
False
>>> f.path.isfile
True
```
A path can be normalized with __normalize()__, and __normalize()__ returns the
Path object for method chaining.
```python
>>> f = furl('http://www.google.com////a/./b/lolsup/../c/')
>>> f.path.normalize()
>>> f.url
'http://www.google.com/a/b/c/'
```
Path segments can also be appended with the slash operator, like with
[pathlib.Path](https://docs.python.org/3/library/pathlib.html#operators).
```python
>>> from __future__ import division # For Python 2.x.
>>>
>>> f = furl('path')
>>> f.path /= 'with'
>>> f.path = f.path / 'more' / 'path segments/'
>>> f.url
'/path/with/more/path%20segments/'
```
For a dictionary representation of a path, use __asdict()__.
```python
>>> f = furl('http://www.google.com/some/enc%20oding')
>>> f.path.asdict()
{ 'encoded': '/some/enc%20oding',
'isabsolute': True,
'isdir': False,
'isfile': True,
'segments': ['some', 'enc oding'] }
```
### Query
URL queries in furl are Query objects that have __params__, a one dimensional
[ordered multivalue dictionary](https://github.com/gruns/orderedmultidict) of
query keys and values. Query keys and values in __params__ are percent-decoded
and all interaction with __params__ should take place with percent-decoded
strings.
```python
>>> f = furl('http://www.google.com/?one=1&two=2')
>>> f.query
Query('one=1&two=2')
>>> f.query.params
omdict1D([('one', '1'), ('two', '2')])
>>> str(f.query)
'one=1&two=2'
```
furl objects and Fragment objects (covered below) contain a Query object, and
__args__ is provided as a shortcut on these objects to access __query.params__.
```python
>>> f = furl('http://www.google.com/?one=1&two=2')
>>> f.query.params
omdict1D([('one', '1'), ('two', '2')])
>>> f.args
omdict1D([('one', '1'), ('two', '2')])
>>> f.args is f.query.params
True
```
#### Manipulation
__params__ is a one dimensional
[ordered multivalue dictionary](https://github.com/gruns/orderedmultidict) that
maintains method parity with Python's standard dictionary.
```python
>>> f.query = 'silicon=14&iron=26&inexorable%20progress=vae%20victus'
>>> f.query.params
omdict1D([('silicon', '14'), ('iron', '26'), ('inexorable progress', 'vae victus')])
>>> del f.args['inexorable progress']
>>> f.args['magnesium'] = '12'
>>> f.args
omdict1D([('silicon', '14'), ('iron', '26'), ('magnesium', '12')])
```
__params__ can also store multiple values for the same key because it's a
multivalue dictionary.
```python
>>> f = furl('http://www.google.com/?space=jams&space=slams')
>>> f.args['space']
'jams'
>>> f.args.getlist('space')
['jams', 'slams']
>>> f.args.addlist('repeated', ['1', '2', '3'])
>>> str(f.query)
'space=jams&space=slams&repeated=1&repeated=2&repeated=3'
>>> f.args.popvalue('space')
'slams'
>>> f.args.popvalue('repeated', '2')
'2'
>>> str(f.query)
'space=jams&repeated=1&repeated=3'
```
__params__ is one dimensional. If a list of values is provided as a query value,
that list is interpreted as multiple values.
```python
>>> f = furl()
>>> f.args['repeated'] = ['1', '2', '3']
>>> f.add(args={'space':['jams', 'slams']})
>>> str(f.query)
'repeated=1&repeated=2&repeated=3&space=jams&space=slams'
```
This makes sense: URL queries are inherently one dimensional -- query values
can't have native subvalues.
See the [orderedmultimdict](https://github.com/gruns/orderedmultidict)
documentation for more information on interacting with the ordered multivalue
dictionary __params__.
#### Parameters
To produce an empty query argument, like `http://sprop.su/?param=`, set the
argument's value to the empty string.
```python
>>> f = furl('http://sprop.su')
>>> f.args['param'] = ''
>>> f.url
'http://sprop.su/?param='
```
To produce an empty query argument without a trailing `=`, use `None` as the
parameter value.
```python
>>> f = furl('http://sprop.su')
>>> f.args['param'] = None
>>> f.url
'http://sprop.su/?param'
```
__encode(delimiter='&', quote_plus=True, dont_quote='')__ can be used to encode
query strings with delimiters like `;`, encode spaces as `+` instead of `%20`
(i.e. application/x-www-form-urlencoded encoded), or avoid percent-encoding
valid query characters entirely (valid query characters are
`/?:@-._~!$&'()*+,;=`).
```python
>>> f.query = 'space=jams&woofs=squeeze+dog'
>>> f.query.encode()
'space=jams&woofs=squeeze+dog'
>>> f.query.encode(';')
'space=jams;woofs=squeeze+dog'
>>> f.query.encode(quote_plus=False)
'space=jams&woofs=squeeze%20dog'
```
`dont_quote` accepts `True`, `False`, or a string of valid query characters to
not percent-enode. If `True`, all valid query characters `/?:@-._~!$&'()*+,;=`
aren't percent-encoded.
```python
>>> f.query = 'one,two/three'
>>> f.query.encode()
'one%2Ctwo%2Fthree'
>>> f.query.encode(dont_quote=True)
'one,two/three'
>>> f.query.encode(dont_quote=',')
'one,two%2Fthree'
```
For a dictionary representation of a query, use __asdict()__.
```python
>>> f = furl('http://www.google.com/?space=ja+ms&space=slams')
>>> f.query.asdict()
{ 'encoded': 'space=ja+ms&space=slams',
'params': [('space', 'ja ms'),
('space', 'slams')] }
```
### Fragment
URL fragments in furl are Fragment objects that have a Path __path__ and Query
__query__ separated by an optional `?` __separator__.
```python
>>> f = furl('http://www.google.com/#/fragment/path?with=params')
>>> f.fragment
Fragment('/fragment/path?with=params')
>>> f.fragment.path
Path('/fragment/path')
>>> f.fragment.query
Query('with=params')
>>> f.fragment.separator
True
```
Manipulation of Fragments is done via the Fragment's Path and Query instances,
__path__ and __query__.
```python
>>> f = furl('http://www.google.com/#/fragment/path?with=params')
>>> str(f.fragment)
'/fragment/path?with=params'
>>> f.fragment.path.segments.append('file.ext')
>>> str(f.fragment)
'/fragment/path/file.ext?with=params'
>>> f = furl('http://www.google.com/#/fragment/path?with=params')
>>> str(f.fragment)
'/fragment/path?with=params'
>>> f.fragment.args['new'] = 'yep'
>>> str(f.fragment)
'/fragment/path?new=yep&with=params'
```
Creating hash-bang fragments with furl illustrates the use of Fragment's boolean
attribute __separator__. When __separator__ is False, the `?` that separates
__path__ and __query__ isn't included.
```python
>>> f = furl('http://www.google.com/')
>>> f.fragment.path = '!'
>>> f.fragment.args = {'a':'dict', 'of':'args'}
>>> f.fragment.separator
True
>>> str(f.fragment)
'!?a=dict&of=args'
>>> f.fragment.separator = False
>>> str(f.fragment)
'!a=dict&of=args'
>>> f.url
'http://www.google.com/#!a=dict&of=args'
```
For a dictionary representation of a fragment, use __asdict()__.
```python
>>> f = furl('http://www.google.com/#path?args=args')
>>> f.fragment.asdict()
{ 'encoded': 'path?args=args',
'separator': True,
'path': { 'encoded': 'path',
'isabsolute': False,
'isdir': False,
'isfile': True,
'segments': ['path']},
'query': { 'encoded': 'args=args',
'params': [('args', 'args')]} }
```
### Encoding
Furl handles encoding for you, and furl's philosophy on encoding is simple: raw
URL strings should always be percent-encoded.
```python
>>> f = furl()
>>> f.netloc = '%40user:%3Apass@google.com'
>>> f.username, f.password
'@user', ':pass'
>>> f = furl()
>>> f.path = 'supply%20percent%20encoded/path%20strings'
>>> f.path.segments
['supply percent encoded', 'path strings']
>>> f.set(query='supply+percent+encoded=query+strings,+too')
>>> f.query.params
omdict1D([('supply percent encoded', 'query strings, too')])
>>> f.set(fragment='percent%20encoded%20path?and+percent+encoded=query+too')
>>> f.fragment.path.segments
['percent encoded path']
>>> f.fragment.args
omdict1D([('and percent encoded', 'query too')])
```
Raw, non-URL strings should never be percent-encoded.
```python
>>> f = furl('http://google.com')
>>> f.set(username='@prap', password=':porps')
>>> f.url
'http://%40prap:%3Aporps@google.com'
>>> f = furl()
>>> f.set(path=['path segments are', 'decoded', '<>[]"#'])
>>> str(f.path)
'/path%20segments%20are/decoded/%3C%3E%5B%5D%22%23'
>>> f.set(args={'query parameters':'and values', 'are':'decoded, too'})
>>> str(f.query)
'query+parameters=and+values&are=decoded,+too'
>>> f.fragment.path.segments = ['decoded', 'path segments']
>>> f.fragment.args = {'and decoded':'query parameters and values'}
>>> str(f.fragment)
'decoded/path%20segments?and+decoded=query+parameters+and+values'
```
Python's
[urllib.quote()](http://docs.python.org/library/urllib.html#urllib.quote) and
[urllib.unquote()](http://docs.python.org/library/urllib.html#urllib.unquote)
can be used to percent-encode and percent-decode path strings. Similarly,
[urllib.quote_plus()](http://docs.python.org/library/urllib.html#urllib.quote_plus)
and
[urllib.unquote_plus()](http://docs.python.org/library/urllib.html#urllib.unquote_plus)
can be used to percent-encode and percent-decode query strings.
### Inline manipulation
For quick, single-line URL manipulation, the __add()__, __set()__, and
__remove()__ methods of furl objects manipulate various URL components and
return the furl object for method chaining.
```python
>>> url = 'http://www.google.com/#fragment'
>>> furl(url).add(args={'example':'arg'}).set(port=99).remove(fragment=True).url
'http://www.google.com:99/?example=arg'
```
__add()__ adds items to a furl object with the optional arguments
* __args__: Shortcut for __query_params__.
* __path__: A list of path segments to add to the existing path segments, or a
path string to join with the existing path string.
* __query_params__: A dictionary of query keys and values to add to the query.
* __fragment_path__: A list of path segments to add to the existing fragment
path segments, or a path string to join with the existing fragment path
string.
* __fragment_args__: A dictionary of query keys and values to add to the
fragment's query.
```python
>>> f = furl('http://www.google.com/').add(
... path='/search', fragment_path='frag/path', fragment_args={'frag':'arg'})
>>> f.url
'http://www.google.com/search#frag/path?frag=args'
```
__set()__ sets items of a furl object with the optional arguments
* __args__: Shortcut for __query_params__.
* __path__: List of path segments or a path string to adopt.
* __scheme__: Scheme string to adopt.
* __netloc__: Network location string to adopt.
* __origin__: Origin string to adopt.
* __query__: Query string to adopt.
* __query_params__: A dictionary of query keys and values to adopt.
* __fragment__: Fragment string to adopt.
* __fragment_path__: A list of path segments to adopt for the fragment's path
or a path string to adopt as the fragment's path.
* __fragment_args__: A dictionary of query keys and values for the fragment's
query to adopt.
* __fragment_separator__: Boolean whether or not there should be a `?`
separator between the fragment path and the fragment query.
* __host__: Host string to adopt.
* __port__: Port number to adopt.
* __username__: Username string to adopt.
* __password__: password string to adopt.
```python
>>> f = furl().set(
... scheme='https', host='secure.google.com', port=99, path='index.html',
... args={'some':'args'}, fragment='great job')
>>> f.url
'https://secure.google.com:99/index.html?some=args#great%20job'
```
__remove()__ removes items from a furl object with the optional arguments
* __args__: Shortcut for __query_params__.
* __path__: A list of path segments to remove from the end of the existing path
segments list, or a path string to remove from the end of the existing
path string, or True to remove the entire path portion of the URL.
* __query__: A list of query keys to remove from the query, if they exist, or
True to remove the entire query portion of the URL.
* __query_params__: A list of query keys to remove from the query, if they
exist.
* __fragment__: If True, remove the entire fragment portion of the URL.
* __fragment_path__: A list of path segments to remove from the end of the
fragment's path segments, or a path string to remove from the end of the
fragment's path string, or True to remove the entire fragment path.
* __fragment_args__: A list of query keys to remove from the fragment's query,
if they exist.
* __username__: If True, remove the username, if it exists.
* __password__: If True, remove the password, if it exists.
```python
>>> url = 'https://secure.google.com:99/a/path/?some=args#great job'
>>> furl(url).remove(args=['some'], path='path/', fragment=True, port=True).url
'https://secure.google.com/a/'
```
### Miscellaneous
Like [pathlib.Path](https://docs.python.org/3/library/pathlib.html#operators),
path segments can be appended to a furl object's Path with the slash operator.
```python
>>> from __future__ import division # For Python 2.x.
>>> f = furl('http://www.google.com/path?example=arg#frag')
>>> f /= 'add'
>>> f = f / 'seg ments/'
>>> f.url
'http://www.google.com/path/add/seg%20ments/?example=arg#frag'
```
__tostr(query_delimiter='&', query_quote_plus=True, query_dont_quote='')__
creates and returns a URL string. `query_delimiter`, `query_quote_plus`, and
`query_dont_quote` are passed unmodified to `Query.encode()` as `delimiter`,
`quote_plus`, and `dont_quote` respectively.
```python
>>> f = furl('http://spep.ru/?a+b=c+d&two%20tap=cat%20nap%24')
>>> f.tostr()
'http://spep.ru/?a+b=c+d&two+tap=cat+nap$'
>>> f.tostr(query_delimiter=';', query_quote_plus=False)
'http://spep.ru/?a%20b=c%20d;two%20tap=cat%20nap$'
>>> f.tostr(query_dont_quote='$')
'http://spep.ru/?a+b=c+d&two+tap=cat+nap$'
```
`furl.url` is a shortcut for `furl.tostr()`.
```python
>>> f.url
'http://spep.ru/?a+b=c+d&two+tap=cat+nap$'
>>> f.url == f.tostr() == str(f)
True
```
__copy()__ creates and returns a new furl object with an identical URL.
```python
>>> f = furl('http://www.google.com')
>>> f.copy().set(path='/new/path').url
'http://www.google.com/new/path'
>>> f.url
'http://www.google.com'
```
__join()__ joins the furl object's URL with the provided relative or absolute
URL and returns the furl object for method chaining. __join()__'s action is the
same as navigating to the provided URL from the current URL in a web browser.
```python
>>> f = furl('http://www.google.com')
>>> f.join('new/path').url
'http://www.google.com/new/path'
>>> f.join('replaced').url
'http://www.google.com/new/replaced'
>>> f.join('../parent').url
'http://www.google.com/parent'
>>> f.join('path?query=yes#fragment').url
'http://www.google.com/path?query=yes#fragment'
>>> f.join('unknown://www.yahoo.com/new/url/').url
'unknown://www.yahoo.com/new/url/'
```
For a dictionary representation of a URL, use __asdict()__.
```python
>>> f = furl('https://xn--eckwd4c7c.xn--zckzah/path?args=args#frag')
>>> f.asdict()
{ 'url': 'https://xn--eckwd4c7c.xn--zckzah/path?args=args#frag',
'scheme': 'https',
'username': None
'password': None,
'host': 'ドメイン.テスト',
'host_encoded': 'xn--eckwd4c7c.xn--zckzah',
'port': 443,
'netloc': 'xn--eckwd4c7c.xn--zckzah',
'origin': 'https://xn--eckwd4c7c.xn--zckzah',
'path': { 'encoded': '/path',
'isabsolute': True,
'isdir': False,
'isfile': True,
'segments': ['path']},
'query': { 'encoded': 'args=args',
'params': [('args', 'args')]},
'fragment': { 'encoded': 'frag',
'path': { 'encoded': 'frag',
'isabsolute': False,
'isdir': False,
'isfile': True,
'segments': ['frag']},
'query': { 'encoded': '',
'params': []},
'separator': True} }
```

126
playground/nlp/spacy.ipynb
Unescape View File

@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 1,
"id": "4b1a1586-0823-4046-ab19-0fa99a0f241b",
"metadata": {
"scrolled": true,
@ -13,73 +13,56 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: spacy in /opt/conda/lib/python3.10/site-packages (3.3.1)\n",
"Requirement already satisfied: pathy>=0.3.5 in /opt/conda/lib/python3.10/site-packages (from spacy) (0.6.1)\n",
"Requirement already satisfied: thinc<8.1.0,>=8.0.14 in /opt/conda/lib/python3.10/site-packages (from spacy) (8.0.17)\n",
"Requirement already satisfied: numpy>=1.15.0 in /opt/conda/lib/python3.10/site-packages (from spacy) (1.21.6)\n",
"Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from spacy) (1.0.2)\n",
"Requirement already satisfied: requests<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from spacy) (2.28.0)\n",
"Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /opt/conda/lib/python3.10/site-packages (from spacy) (3.3.0)\n",
"Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from spacy) (2.0.6)\n",
"Requirement already satisfied: blis<0.8.0,>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from spacy) (0.7.7)\n",
"Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /opt/conda/lib/python3.10/site-packages (from spacy) (1.0.7)\n",
"Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /opt/conda/lib/python3.10/site-packages (from spacy) (3.0.6)\n",
"Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /opt/conda/lib/python3.10/site-packages (from spacy) (2.4.3)\n",
"Requirement already satisfied: packaging>=20.0 in /home/jovyan/.local/lib/python3.10/site-packages (from spacy) (21.3)\n",
"Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /opt/conda/lib/python3.10/site-packages (from spacy) (2.0.7)\n",
"Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /opt/conda/lib/python3.10/site-packages (from spacy) (0.9.1)\n",
"Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /opt/conda/lib/python3.10/site-packages (from spacy) (4.64.0)\n",
"Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from spacy) (62.3.4)\n",
"Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /opt/conda/lib/python3.10/site-packages (from spacy) (3.0.9)\n",
"Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from spacy) (3.1.2)\n",
"Requirement already satisfied: typer<0.5.0,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from spacy) (0.4.1)\n",
"Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /opt/conda/lib/python3.10/site-packages (from spacy) (1.8.2)\n",
"Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->spacy) (3.0.9)\n",
"Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from pathy>=0.3.5->spacy) (5.2.1)\n",
"Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy) (4.2.0)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2022.6.15)\n",
"Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.0.12)\n",
"Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.3)\n",
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.26.9)\n",
"Requirement already satisfied: click<9.0.0,>=7.1.1 in /opt/conda/lib/python3.10/site-packages (from typer<0.5.0,>=0.3.0->spacy) (8.1.3)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->spacy) (2.1.1)\n",
"Collecting en-core-web-sm==3.3.0\n",
" Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl (12.8 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: spacy<3.4.0,>=3.3.0.dev0 in /opt/conda/lib/python3.10/site-packages (from en-core-web-sm==3.3.0) (3.3.1)\n",
"Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.4.3)\n",
"Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.8.2)\n",
"Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.64.0)\n",
"Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.6)\n",
"Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (62.3.4)\n",
"Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
"Requirement already satisfied: typer<0.5.0,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.4.1)\n",
"Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.9.1)\n",
"Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.2)\n",
"Requirement already satisfied: packaging>=20.0 in /home/jovyan/.local/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (21.3)\n",
"Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3.0)\n",
"Requirement already satisfied: blis<0.8.0,>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.7.7)\n",
"Requirement already satisfied: requests<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.28.0)\n",
"Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.6)\n",
"Requirement already satisfied: numpy>=1.15.0 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.21.6)\n",
"Requirement already satisfied: thinc<8.1.0,>=8.0.14 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.0.17)\n",
"Requirement already satisfied: pathy>=0.3.5 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.6.1)\n",
"Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.7)\n",
"Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.1.2)\n",
"Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /opt/conda/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.7)\n",
"Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
"Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from pathy>=0.3.5->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (5.2.1)\n",
"Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.2.0)\n",
"Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.12)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2022.6.15)\n",
"Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3)\n",
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.26.9)\n",
"Requirement already satisfied: click<9.0.0,>=7.1.1 in /opt/conda/lib/python3.10/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.1.3)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.1.1)\n",
"Installing collected packages: en-core-web-sm\n",
"Successfully installed en-core-web-sm-3.3.0\n",
"\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
"You can now load the package via spacy.load('en_core_web_sm')\n"
"Defaulting to user installation because normal site-packages is not writeable\n",
"Collecting spacy\n",
" Downloading spacy-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.4 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.4/6.4 MB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
"\u001b[?25hCollecting thinc<8.2.0,>=8.1.0\n",
" Downloading thinc-8.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (822 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m822.5/822.5 KB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
"\u001b[?25hCollecting cymem<2.1.0,>=2.0.2\n",
" Using cached cymem-2.0.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (35 kB)\n",
"Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/lib/python3/dist-packages (from spacy) (2.25.1)\n",
"Collecting wasabi<1.1.0,>=0.9.1\n",
" Downloading wasabi-0.10.1-py3-none-any.whl (26 kB)\n",
"Requirement already satisfied: packaging>=20.0 in /home/donaldrich/.local/lib/python3.10/site-packages (from spacy) (21.3)\n",
"Collecting spacy-legacy<3.1.0,>=3.0.9\n",
" Using cached spacy_legacy-3.0.9-py2.py3-none-any.whl (20 kB)\n",
"Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from spacy) (3.0.3)\n",
"Collecting langcodes<4.0.0,>=3.2.0\n",
" Using cached langcodes-3.3.0-py3-none-any.whl (181 kB)\n",
"Collecting pathy>=0.3.5\n",
" Downloading pathy-0.6.2-py3-none-any.whl (42 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.8/42.8 KB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting murmurhash<1.1.0,>=0.28.0\n",
" Using cached murmurhash-1.0.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (21 kB)\n",
"Collecting tqdm<5.0.0,>=4.38.0\n",
" Downloading tqdm-4.64.0-py2.py3-none-any.whl (78 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.4/78.4 KB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting preshed<3.1.0,>=3.0.2\n",
" Using cached preshed-3.0.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (128 kB)\n",
"Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from spacy) (59.6.0)\n",
"Collecting spacy-loggers<2.0.0,>=1.0.0\n",
" Downloading spacy_loggers-1.0.3-py3-none-any.whl (9.3 kB)\n",
"Collecting catalogue<2.1.0,>=2.0.6\n",
" Downloading catalogue-2.0.8-py3-none-any.whl (17 kB)\n",
"Requirement already satisfied: typer<0.5.0,>=0.3.0 in /home/donaldrich/.local/lib/python3.10/site-packages (from spacy) (0.4.2)\n",
"Requirement already satisfied: numpy>=1.15.0 in /usr/lib/python3/dist-packages (from spacy) (1.21.5)\n",
"Collecting srsly<3.0.0,>=2.4.3\n",
" Downloading srsly-2.4.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (460 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m460.0/460.0 KB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4 in /home/donaldrich/.local/lib/python3.10/site-packages (from spacy) (1.9.1)\n",
"Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/lib/python3/dist-packages (from packaging>=20.0->spacy) (2.4.7)\n",
"Collecting smart-open<6.0.0,>=5.2.1\n",
" Using cached smart_open-5.2.1-py3-none-any.whl (58 kB)\n",
"Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/donaldrich/.local/lib/python3.10/site-packages (from pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4->spacy) (4.3.0)\n",
"Collecting blis<0.8.0,>=0.7.8\n",
" Downloading blis-0.7.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.2 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.2/10.2 MB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: click<9.0.0,>=7.1.1 in /usr/lib/python3/dist-packages (from typer<0.5.0,>=0.3.0->spacy) (8.0.3)\n",
"Installing collected packages: wasabi, murmurhash, cymem, tqdm, spacy-loggers, spacy-legacy, smart-open, preshed, langcodes, catalogue, blis, srsly, pathy, thinc, spacy\n",
"Successfully installed blis-0.7.8 catalogue-2.0.8 cymem-2.0.6 langcodes-3.3.0 murmurhash-1.0.7 pathy-0.6.2 preshed-3.0.6 smart-open-5.2.1 spacy-3.4.1 spacy-legacy-3.0.9 spacy-loggers-1.0.3 srsly-2.4.4 thinc-8.1.0 tqdm-4.64.0 wasabi-0.10.1\n",
"zsh:1: command not found: python\n"
]
}
],
@ -116,7 +99,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
@ -131,6 +114,11 @@
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
},
"vscode": {
"interpreter": {
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
}
},
"nbformat": 4,

53
storage/minio.ipynb
Unescape View File

@ -0,0 +1,53 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from minio import Minio\n",
"from minio.error import S3Error\n",
"\n",
"\n",
"def main():\n",
"\n",
" client = Minio(\n",
" \"minio_endpoint\",\n",
" access_key=\"key\",\n",
" secret_key=\"secret\",\n",
" )\n",
"\n",
" # Make 'asiatrip' bucket if not exist.\n",
" found = client.bucket_exists(\"bucket\")\n",
" if not found:\n",
" client.make_bucket(\"bucket\")\n",
" else:\n",
" print(\"Bucket already exists\")\n",
"\n",
" client.fput_object(\n",
" \"bucket\", \"bucket_file\", \"local_file_path\",\n",
" )\n",
" print(\n",
" \"'local_file_path' is successfully uploaded as \"\n",
" \"object 'bucket_file' to bucket 'bucket'.\"\n",
" )\n",
"\n",
"\n",
"if __name__ == \"__main__\":\n",
" try:\n",
" main()\n",
" except S3Error as exc:\n",
" print(\"error occurred.\", exc)"
]
}
],
"metadata": {
"language_info": {
"name": "python"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}

26
templating/Dashmachine Config Template.ipynb
Unescape View File

@ -51,7 +51,7 @@
"{% endfor %}\n",
"\n",
"{# a comment #}\n",
"```"
"EOF"
]
},
{
@ -126,13 +126,25 @@
},
{
"cell_type": "code",
"execution_count": 40,
"execution_count": 1,
"id": "ad3a0cde-8e53-403d-bbb6-6fc95c392093",
"metadata": {
"scrolled": true,
"tags": []
},
"outputs": [],
"outputs": [
{
"ename": "KeyError",
"evalue": "'data'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/home/donaldrich/projects/secrets/jupyter-notebooks/templating/Dashmachine Config Template.ipynb Cell 3\u001b[0m in \u001b[0;36m<cell line: 16>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/templating/Dashmachine%20Config%20Template.ipynb#W2sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m response \u001b[39m=\u001b[39m requests\u001b[39m.\u001b[39mget(\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/templating/Dashmachine%20Config%20Template.ipynb#W2sdnNjb2RlLXJlbW90ZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mhttps://cms.donavanaldrich.com/items/containers\u001b[39m\u001b[39m\"\u001b[39m, headers\u001b[39m=\u001b[39mheaders\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/templating/Dashmachine%20Config%20Template.ipynb#W2sdnNjb2RlLXJlbW90ZQ%3D%3D?line=12'>13</a>\u001b[0m )\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/templating/Dashmachine%20Config%20Template.ipynb#W2sdnNjb2RlLXJlbW90ZQ%3D%3D?line=13'>14</a>\u001b[0m data \u001b[39m=\u001b[39m response\u001b[39m.\u001b[39mjson()\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bwebserver/home/donaldrich/projects/secrets/jupyter-notebooks/templating/Dashmachine%20Config%20Template.ipynb#W2sdnNjb2RlLXJlbW90ZQ%3D%3D?line=15'>16</a>\u001b[0m items \u001b[39m=\u001b[39m data[\u001b[39m\"\u001b[39;49m\u001b[39mdata\u001b[39;49m\u001b[39m\"\u001b[39;49m]\n",
"\u001b[0;31mKeyError\u001b[0m: 'data'"
]
}
],
"source": [
"import json\n",
"import os\n",
@ -141,7 +153,7 @@
"import requests\n",
"\n",
"headers = {\n",
" \"Authorization\": \"os.env(directus_token)\",\n",
" \"Authorization\": os.env(directus_token),\n",
" \"Content-Type\": \"application/json\",\n",
"}\n",
"response = requests.get(\n",
@ -1533,7 +1545,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9.13 64-bit",
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
@ -1547,11 +1559,11 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
"version": "3.10.4"
},
"vscode": {
"interpreter": {
"hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
}
},

15
webscraping/pyppeteer/walgreens.ipynb
Unescape View File

@ -9,10 +9,12 @@
]
},
{
"cell_type": "raw",
"id": "d5ddb5e6-f653-43c8-958f-e0d249b8b30d",
"cell_type": "markdown",
"id": "82fee814",
"metadata": {},
"source": [
"\n",
"```javascript\n",
"export default async ({ page }: { page: Page }) => {\n",
" const navigationPromise = page.waitForNavigation()\n",
"\n",
@ -31,13 +33,14 @@
" await page.waitForNavigation()\n",
" await page.screenshot({ path: screenshot })\n",
"\n",
"};"
"};\n",
"```"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9.13 64-bit",
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
@ -51,11 +54,11 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
"version": "3.10.4"
},
"vscode": {
"interpreter": {
"hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
}
},

174
webscraping/rendering/splash.ipynb
View File

File diff suppressed because one or more lines are too long

40
webscraping/robot_framework/Untitled.ipynb
Unescape View File

@ -11,22 +11,9 @@
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "raw",
"id": "d2f5ca23-5eed-4eb6-9c1b-2152b08009a5",
"metadata": {},
"outputs": [
{
"ename": "",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[1;31mFailed to start the Kernel. \n",
"Kernel Python 3.9.13 64-bit is not usable. Check the Jupyter output tab for more information. \n",
"View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
]
}
],
"source": [
"*** Settings ***\n",
"Documentation Simple example using SeleniumLibrary.\n",
@ -64,30 +51,11 @@
"Welcome Page Should Be Open\n",
" Title Should Be Welcome Page"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "78a7b507-e8f5-4dab-9d38-fb5396e92b42",
"metadata": {},
"outputs": [
{
"ename": "",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[1;31mFailed to start the Kernel. \n",
"Kernel Python 3.9.13 64-bit is not usable. Check the Jupyter output tab for more information. \n",
"View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
]
}
],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9.13 64-bit",
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
@ -101,11 +69,11 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
"version": "3.10.4"
},
"vscode": {
"interpreter": {
"hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
}
},

Write Preview
Loading…
Cancel
Save