From 74cab757af50b01cbdab36572f0ef97ce1828fb7 Mon Sep 17 00:00:00 2001
From: Amir Zarrinkafsh <nightah@me.com>
Date: Thu, 29 Oct 2020 23:05:45 +1100
Subject: [PATCH 1/7] Fix generation scripts
This change leverages #130 and also applies this to the dnsmasq script.
As it currently stands both generation scripts (unbound and dnsmasq) have a condition where a domain will be skipped if it fuzzy matches a domain already parsed that is higher in the CDN domain list.
For example the latter of the below two samples would never be added.
https://github.com/uklans/cache-domains/blob/8793ce15315cac1e594f7602158c2e82f510bc91/steam.txt#L20 https://github.com/uklans/cache-domains/blob/8793ce15315cac1e594f7602158c2e82f510bc91/steam.txt#L29
I've also taken the liberty to sort the output of said scripts for readability and troubleshooting purposes.
Closes #130.
---
scripts/create-dnsmasq.sh | 20 ++++++++++----------
scripts/create-unbound.sh | 4 ++--
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/scripts/create-dnsmasq.sh b/scripts/create-dnsmasq.sh
index 559fd59..abe94b0 100755
--- a/scripts/create-dnsmasq.sh
+++ b/scripts/create-dnsmasq.sh
@@ -50,32 +50,32 @@ while read -r entry; do
touch "$outputfile"
# Wildcard entries
while read -r fileentry; do
- # Ignore comments
- if [[ $fileentry == \#* ]]; then
+ # Ignore comments and non-wildcards
+ if [[ $fileentry == \#* ]] || [[ ! $fileentry =~ ^\*\. ]]; then
continue
fi
- wildcard=$(echo $fileentry | grep "*." | sed -e "s/^\*\.//")
- if grep -q "$wildcard" "$lancacheconf"; then
+ wildcard=$(echo $fileentry | sed -e "s/^\*\.//")
+ if grep -qx "$wildcard" "$lancacheconf"; then
continue
fi
for i in ${cacheip}; do
echo "address=/${wildcard}/${i}" >> "$lancacheconf"
done
- done <<< $(cat ${basedir}/$filename);
+ done <<< $(cat ${basedir}/$filename | sort);
# All other entries
while read -r fileentry; do
- # Ignore comments
- if [[ $fileentry == \#* ]]; then
+ # Ignore comments and wildcards
+ if [[ $fileentry =~ ^(\#|\*\.) ]]; then
continue
fi
- parsed=$(echo $fileentry | sed -e "s/^\*\.//")
- if grep -q "$parsed" "$outputfile"; then
+ parsed=$(echo $fileentry)
+ if grep -qx "$parsed" "$outputfile"; then
continue
fi
for i in ${cacheip}; do
echo "${i} ${parsed}" >> "$outputfile"
done
- done <<< $(cat ${basedir}/$filename);
+ done <<< $(cat ${basedir}/$filename | sort);
done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
diff --git a/scripts/create-unbound.sh b/scripts/create-unbound.sh
index 10ee026..1364a47 100755
--- a/scripts/create-unbound.sh
+++ b/scripts/create-unbound.sh
@@ -52,14 +52,14 @@ while read entry; do
continue
fi
parsed=$(echo $fileentry | sed -e "s/^\*\.//")
- if grep -q "$parsed" $outputfile; then
+ if grep -qx "$parsed" $outputfile; then
continue
fi
echo " local-zone: \"${parsed}\" redirect" >> $outputfile
for i in ${cacheip}; do
echo " local-data: \"${parsed} 30 IN A ${i}\"" >> $outputfile
done
- done <<< $(cat ${basedir}/$filename);
+ done <<< $(cat ${basedir}/$filename | sort);
done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
From 3e649ab5d7922a70263c2f25b366b7d4282e2dd3 Mon Sep 17 00:00:00 2001
From: Amir Zarrinkafsh <nightah@me.com>
Date: Fri, 15 Jan 2021 08:48:31 +1100
Subject: [PATCH 2/7] Add README.md for DNS generation scripts
Also fix up some typos in the main README.md.
---
README.md | 10 +++++-----
scripts/README.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 54 insertions(+), 5 deletions(-)
create mode 100755 scripts/README.md
diff --git a/README.md b/README.md
index ffb506d..20ae7c9 100755
--- a/README.md
+++ b/README.md
@@ -11,13 +11,13 @@ You can use this list one of two ways:
- Overriding DNS for these hostnames to point to the IP of your cache server.
- Use them in Squid with WCCP to redirect content to the right cache server.
-There is a cache_domains.json file to define CDNs and additional meta deta with the following structure
+There is a cache_domains.json file to define CDNs and additional metadata with the following structure
- cache_domains: Array of cache_domain object
- name: shortname for the cache domain. Should match `^[0-9A-Za-z]$`
- description: a longer description to aid others in identifying what this domain does (not all users of this repo will want to enable all caches)
- notes: implementation specific notes which may be useful for other users
- - domain_files: array of files within the repo assosciated to the cdn. Most cdn's only need one file
+ - domain_files: array of files within the repo associated to the cdn. Most CDNs only need one file
- Example domain entry for steam
```json
{
@@ -35,7 +35,7 @@ There is a cache_domains.json file to define CDNs and additional meta deta with
There is a separate file for each cacheable service. Some notes on formatting:
- - Every line should be a seperate hostname for that service.
+ - Every line should be a separate hostname for that service.
- Only one entry is permitted per line.
- Wildcards are permitted as per below
- Lines starting with a # will be treated as a comment.
@@ -46,10 +46,10 @@ There is a separate file for each cacheable service. Some notes on formatting:
The wildcard format shall be defined as per the below
- - Wildcards should be represented with an asterix.
+ - Wildcards should be represented with an asterisk.
- If a wildcard is used, it should be the first character on the line.
- Wildcards are not treated as matching null, e.g. `*.example.com` will match `a.example.com` but will not match `example.com`
- - Only simple domain wildcards will be accepted eg `*.example.com` not `*ww.example.com`
+ - Only simple domain wildcards will be accepted e.g. `*.example.com` not `*ww.example.com`
##### Notes for Squid users
diff --git a/scripts/README.md b/scripts/README.md
new file mode 100755
index 0000000..32f3757
--- /dev/null
+++ b/scripts/README.md
@@ -0,0 +1,49 @@
+# DNS Generation Scripts
+
+## Introduction
+
+The respective shell scripts contained within this directory can be utilised to generate application specific compliant
+configuration which can be utilised with:
+
+* Dnsmasq
+* Unbound
+
+## Usage
+
+1. Copy `config.example.json` to `config.json`.
+2. Modify `config.json` to include your Cacheserver's IP(s) and the CDNs you plan to cache.
+ The following example assumes a single shared Cacheserver IP:
+```json
+{
+ "ips": {
+ "generic": ["10.10.10.200"]
+ },
+ "cache_domains": {
+ "blizzard": "generic",
+ "epicgames": "generic",
+ "nintendo": "generic",
+ "origin": "generic",
+ "riot": "generic",
+ "sony": "generic",
+ "steam": "generic",
+ "uplay": "generic",
+ "wsus": "generic"
+ }
+}
+```
+3. Run generation script relative to your DNS implementation: `bash create-dnsmasq.sh`.
+4. Copy files from `output/{dnsmasq,unbound}/*` to the respective locations for Dnsmasq/Unbound.
+5. Restart Dnsmasq or Unbound.
+
+### Notes for Dnsmasq users
+
+**This also applies to users utilising the script alongside Pi-hole.**
+
+If utilising the `create-dnsmasq.sh` the generation script will create a `lancache.conf` which also loads in the respective `*.hosts` files.
+
+The `lancache.conf` should be copied into the `/etc/dnsmasq.d/` location but also will need to be modified to point to the respective location of the `*.hosts` files.
+
+You can copy the `*.hosts` file to any location other than `/etc/dnsmasq.d/` as this location is utilised only for `*.conf` files.
+
+For example if utilising Pi-hole a user can copy the `*.hosts` files to `/etc/pihole/` and modify the `lancache.conf` with the following command, prior to copying it to `/etc/dnsmasq.d/`:
+`sed -i 's/dnsmasq.d/pihole/g' output/dnsmasq/lancache.conf`
\ No newline at end of file
From 654d19ce849fde1698f23e6ea2cb15dc31bb5738 Mon Sep 17 00:00:00 2001
From: Amir Zarrinkafsh <nightah@me.com>
Date: Fri, 15 Jan 2021 10:20:51 +1100
Subject: [PATCH 3/7] Change Dnsmasq *.hosts reference to /etc/dnsmasq/
---
scripts/create-dnsmasq.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/create-dnsmasq.sh b/scripts/create-dnsmasq.sh
index abe94b0..bf5eb81 100755
--- a/scripts/create-dnsmasq.sh
+++ b/scripts/create-dnsmasq.sh
@@ -46,7 +46,7 @@ while read -r entry; do
destfilename=$(echo $filename | sed -e 's/txt/hosts/')
lancacheconf=${outputdir}/lancache.conf
outputfile=${outputdir}/${destfilename}
- echo "addn-hosts=/etc/dnsmasq.d/${destfilename}" >> ${lancacheconf}
+ echo "addn-hosts=/etc/dnsmasq/${destfilename}" >> ${lancacheconf}
touch "$outputfile"
# Wildcard entries
while read -r fileentry; do
From c05d2cfec412b18b99ea6f9cd2781ec623725e23 Mon Sep 17 00:00:00 2001
From: Amir Zarrinkafsh <nightah@me.com>
Date: Fri, 15 Jan 2021 19:01:02 +1100
Subject: [PATCH 4/7] Output *.hosts file in output/dnsmasq/hosts/ directory
---
scripts/README.md | 2 +-
scripts/create-dnsmasq.sh | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/scripts/README.md b/scripts/README.md
index 32f3757..96fcc19 100755
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -46,4 +46,4 @@ The `lancache.conf` should be copied into the `/etc/dnsmasq.d/` location but als
You can copy the `*.hosts` file to any location other than `/etc/dnsmasq.d/` as this location is utilised only for `*.conf` files.
For example if utilising Pi-hole a user can copy the `*.hosts` files to `/etc/pihole/` and modify the `lancache.conf` with the following command, prior to copying it to `/etc/dnsmasq.d/`:
-`sed -i 's/dnsmasq.d/pihole/g' output/dnsmasq/lancache.conf`
\ No newline at end of file
+`sed -i 's/dnsmasq\/hosts/pihole/g' output/dnsmasq/lancache.conf`
\ No newline at end of file
diff --git a/scripts/create-dnsmasq.sh b/scripts/create-dnsmasq.sh
index bf5eb81..6329f4e 100755
--- a/scripts/create-dnsmasq.sh
+++ b/scripts/create-dnsmasq.sh
@@ -26,7 +26,7 @@ while read -r line; do
done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
rm -rf ${outputdir}
-mkdir -p ${outputdir}
+mkdir -p ${outputdir}/hosts
touch ${outputdir}/lancache.conf
while read -r entry; do
unset cacheip
@@ -45,8 +45,8 @@ while read -r entry; do
while read -r filename; do
destfilename=$(echo $filename | sed -e 's/txt/hosts/')
lancacheconf=${outputdir}/lancache.conf
- outputfile=${outputdir}/${destfilename}
- echo "addn-hosts=/etc/dnsmasq/${destfilename}" >> ${lancacheconf}
+ outputfile=${outputdir}/hosts/${destfilename}
+ echo "addn-hosts=/etc/dnsmasq/hosts/${destfilename}" >> ${lancacheconf}
touch "$outputfile"
# Wildcard entries
while read -r fileentry; do
From 634b7dfff83f12cefb158f5681ce836cad7d9f69 Mon Sep 17 00:00:00 2001
From: Amir Zarrinkafsh <nightah@me.com>
Date: Fri, 15 Jan 2021 20:09:32 +1100
Subject: [PATCH 5/7] Provide advice to users upon completion of script
---
scripts/create-dnsmasq.sh | 2 ++
1 file changed, 2 insertions(+)
diff --git a/scripts/create-dnsmasq.sh b/scripts/create-dnsmasq.sh
index 6329f4e..3a2d5a9 100755
--- a/scripts/create-dnsmasq.sh
+++ b/scripts/create-dnsmasq.sh
@@ -79,3 +79,5 @@ while read -r entry; do
done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
+
+echo "Please copy the following files:\n- ./output/dnsmasq/lancache.conf to /etc/dnsmasq/dnsmasq.d/\n- ./output/dnsmasq/hosts to /etc/dnsmasq/"
From 14ccfecb1f26006b8f9ddb2dedb969ca072c49e0 Mon Sep 17 00:00:00 2001
From: Amir Zarrinkafsh <nightah@me.com>
Date: Fri, 15 Jan 2021 21:43:21 +1100
Subject: [PATCH 6/7] Include advice for unbound script
---
scripts/create-dnsmasq.sh | 8 +++++++-
scripts/create-unbound.sh | 7 +++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/scripts/create-dnsmasq.sh b/scripts/create-dnsmasq.sh
index 3a2d5a9..b3e054d 100755
--- a/scripts/create-dnsmasq.sh
+++ b/scripts/create-dnsmasq.sh
@@ -80,4 +80,10 @@ while read -r entry; do
done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
-echo "Please copy the following files:\n- ./output/dnsmasq/lancache.conf to /etc/dnsmasq/dnsmasq.d/\n- ./output/dnsmasq/hosts to /etc/dnsmasq/"
+cat << EOF
+Configuration generation completed.
+
+Please copy the following files:
+- ./${outputdir}/lancache.conf to /etc/dnsmasq/dnsmasq.d/
+- ./${outputdir}/hosts to /etc/dnsmasq/
+EOF
diff --git a/scripts/create-unbound.sh b/scripts/create-unbound.sh
index 1364a47..32f64c3 100755
--- a/scripts/create-unbound.sh
+++ b/scripts/create-unbound.sh
@@ -63,3 +63,10 @@ while read entry; do
done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
+
+cat << EOF
+Configuration generation completed.
+
+Please copy the following files:
+- ./${outputdir}/*.conf to /etc/unbound/unbound.conf.d/
+EOF
From b9a669ef5ae695896e40c6a0201dad2047e2be37 Mon Sep 17 00:00:00 2001
From: James Kinsman <VibroAxe@users.noreply.github.com>
Date: Fri, 15 Jan 2021 12:15:28 +0000
Subject: [PATCH 7/7] Update .gitignore
Ignore the users config.json for script generation
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index 86af95d..dfa9afb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
scripts/output
+scripts/config.json