Moved python code to separate script
moved python code to separate script
This commit is contained in:
@@ -12,58 +12,7 @@ jobs:
|
|||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
- name: Fetch and merge upstream files
|
- name: Fetch and merge upstream files
|
||||||
run: |
|
run: python3 scripts/merge_blocklists.py
|
||||||
python3 << 'PYEOF'
|
|
||||||
import urllib.request
|
|
||||||
import os
|
|
||||||
|
|
||||||
files = {
|
|
||||||
"blacklist": "https://raw.githubusercontent.com/Cleanuparr/Cleanuparr/main/blacklist",
|
|
||||||
"blacklist_permissive": "https://raw.githubusercontent.com/Cleanuparr/Cleanuparr/main/blacklist_permissive",
|
|
||||||
"whitelist": "https://raw.githubusercontent.com/Cleanuparr/Cleanuparr/main/whitelist",
|
|
||||||
"whitelist_with_subtitles": "https://raw.githubusercontent.com/Cleanuparr/Cleanuparr/main/whitelist_with_subtitles",
|
|
||||||
}
|
|
||||||
|
|
||||||
def merge_blocklist(filename, url):
|
|
||||||
prev_file = f"{filename}.prev"
|
|
||||||
|
|
||||||
# Fetch new upstream
|
|
||||||
with urllib.request.urlopen(url) as r:
|
|
||||||
upstream_new = set(line.strip() for line in r.read().decode().splitlines() if line.strip())
|
|
||||||
|
|
||||||
# Read previous upstream (empty set if first run)
|
|
||||||
try:
|
|
||||||
with open(prev_file) as f:
|
|
||||||
upstream_prev = set(line.strip() for line in f if line.strip())
|
|
||||||
except FileNotFoundError:
|
|
||||||
upstream_prev = upstream_new.copy()
|
|
||||||
|
|
||||||
# Read current local file
|
|
||||||
try:
|
|
||||||
with open(filename) as f:
|
|
||||||
local = set(line.strip() for line in f if line.strip())
|
|
||||||
except FileNotFoundError:
|
|
||||||
local = set()
|
|
||||||
|
|
||||||
# Three-way merge
|
|
||||||
custom = local - upstream_prev # your additions
|
|
||||||
result = upstream_new | custom # new upstream + your custom entries
|
|
||||||
|
|
||||||
print(f"[{filename}] Custom preserved: {sorted(custom)}")
|
|
||||||
print(f"[{filename}] Upstream added: {sorted(upstream_new - upstream_prev)}")
|
|
||||||
print(f"[{filename}] Upstream removed: {sorted(upstream_prev - upstream_new)}")
|
|
||||||
|
|
||||||
# Write merged result sorted
|
|
||||||
with open(filename, "w") as f:
|
|
||||||
f.write("\n".join(sorted(result)) + "\n")
|
|
||||||
|
|
||||||
# Store new upstream as prev for next run
|
|
||||||
with open(prev_file, "w") as f:
|
|
||||||
f.write("\n".join(sorted(upstream_new)) + "\n")
|
|
||||||
|
|
||||||
for filename, url in files.items():
|
|
||||||
merge_blocklist(filename, url)
|
|
||||||
PYEOF
|
|
||||||
|
|
||||||
- name: Commit and push if changed
|
- name: Commit and push if changed
|
||||||
run: |
|
run: |
|
||||||
@@ -72,14 +21,3 @@ jobs:
|
|||||||
git add .
|
git add .
|
||||||
git diff --staged --quiet || git commit -m "Sync blocklists from upstream"
|
git diff --staged --quiet || git commit -m "Sync blocklists from upstream"
|
||||||
git push
|
git push
|
||||||
```
|
|
||||||
|
|
||||||
**Step 4 — Trigger manual run**
|
|
||||||
|
|
||||||
Go to the Actions tab in your repo and trigger `workflow_dispatch` to verify the first run works and the `.prev` files are created.
|
|
||||||
|
|
||||||
**Step 5 — Update Cleanuparr**
|
|
||||||
|
|
||||||
Point Cleanuparr's Malware Blocker and Blacklist Sync at your Gitea raw URL:
|
|
||||||
```
|
|
||||||
https://git.hisp.no/arr/blocklists/raw/branch/main/blacklist_permissive
|
|
||||||
Reference in New Issue
Block a user