Merge: * DNS: allow-filters: support updating

Close #1435

* commit '897ff436b1920a857b8881db7b213acecc7c9f85':
  * openapi: change /filtering/refresh
  + client: handle allowlist refresh
  * allow-filters: support updating

AGHTechDoc.md

@@ -1405,6 +1405,10 @@ Request:
 
 	POST /control/filtering/refresh
 
+	{
+		"whitelist": true
+	}
+
 Response:
 
 	200 OK

client/src/actions/filtering.js

@@ -113,11 +113,11 @@ export const refreshFiltersRequest = createAction('FILTERING_REFRESH_REQUEST');
 export const refreshFiltersFailure = createAction('FILTERING_REFRESH_FAILURE');
 export const refreshFiltersSuccess = createAction('FILTERING_REFRESH_SUCCESS');
 
-export const refreshFilters = () => async (dispatch) => {
+export const refreshFilters = config => async (dispatch) => {
     dispatch(refreshFiltersRequest());
     dispatch(showLoading());
     try {
-        const data = await apiClient.refreshFilters();
+        const data = await apiClient.refreshFilters(config);
         const { updated } = data;
         dispatch(refreshFiltersSuccess());
 

client/src/api/Api.js

@@ -89,9 +89,14 @@ class Api {
         return this.makeRequest(path, method);
     }
 
-    refreshFilters() {
+    refreshFilters(config) {
         const { path, method } = this.FILTERING_REFRESH;
-        return this.makeRequest(path, method);
+        const parameters = {
+            data: config,
+            headers: { 'Content-Type': 'application/json' },
+        };
+
+        return this.makeRequest(path, method, parameters);
     }
 
     addFilter(config) {

client/src/components/Filters/DnsAllowlist.js

@@ -40,11 +40,14 @@ class DnsAllowlist extends Component {
         this.props.toggleFilterStatus(url, data, whitelist);
     };
 
+    handleRefresh = () => {
+        this.props.refreshFilters({ whitelist: true });
+    };
+
     render() {
         const {
             t,
             toggleFilteringModal,
-            refreshFilters,
             addFilter,
             toggleFilterStatus,
             filtering: {
@@ -89,7 +92,7 @@ class DnsAllowlist extends Component {
                                 />
                                 <Actions
                                     handleAdd={() => toggleFilteringModal({ type: MODAL_TYPE.ADD })}
-                                    handleRefresh={refreshFilters}
+                                    handleRefresh={this.handleRefresh}
                                     processingRefreshFilters={processingRefreshFilters}
                                     whitelist={whitelist}
                                 />

client/src/components/Filters/DnsBlocklist.js

@@ -37,11 +37,14 @@ class DnsBlocklist extends Component {
         this.props.toggleFilterStatus(url, data);
     };
 
+    handleRefresh = () => {
+        this.props.refreshFilters({ whitelist: false });
+    };
+
     render() {
         const {
             t,
             toggleFilteringModal,
-            refreshFilters,
             addFilter,
             filtering: {
                 filters,
@@ -82,7 +85,7 @@ class DnsBlocklist extends Component {
                                 />
                                 <Actions
                                     handleAdd={() => toggleFilteringModal({ type: MODAL_TYPE.ADD })}
-                                    handleRefresh={refreshFilters}
+                                    handleRefresh={this.handleRefresh}
                                     processingRefreshFilters={processingRefreshFilters}
                                 />
                             </Card>

home/control_filtering.go

@@ -185,17 +185,9 @@ func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
 	}
 
 	onConfigModified()
-	if (status & statusURLChanged) != 0 {
+	if (status&(statusURLChanged|statusEnabledChanged)) != 0 && fj.Data.Enabled {
-		if fj.Data.Enabled {
+		// download new filter and apply its rules
-			// download new filter and apply its rules
+		_, _ = refreshFilters(fj.Whitelist, true)
-			refreshStatus = 1
-			refreshLock.Lock()
-			_, _ = refreshFiltersIfNecessary(true)
-			refreshLock.Unlock()
-		}
-
-	} else if (status & statusEnabledChanged) != 0 {
-		enableFilters(true)
 	}
 }
 
@@ -217,14 +209,24 @@ func handleFilteringSetRules(w http.ResponseWriter, r *http.Request) {
 }
 
 func handleFilteringRefresh(w http.ResponseWriter, r *http.Request) {
+	type Req struct {
+		White bool `json:"whitelist"`
+	}
 	type Resp struct {
 		Updated int `json:"updated"`
 	}
 	resp := Resp{}
 	var err error
 
+	req := Req{}
+	err = json.NewDecoder(r.Body).Decode(&req)
+	if err != nil {
+		httpError(w, http.StatusBadRequest, "json decode: %s", err)
+		return
+	}
+
 	Context.controlLock.Unlock()
-	resp.Updated, err = refreshFilters()
+	resp.Updated, err = refreshFilters(req.White, false)
 	Context.controlLock.Lock()
 	if err != nil {
 		httpError(w, http.StatusInternalServerError, "%s", err)

home/filter.go

@@ -237,7 +237,7 @@ func periodicallyRefreshFilters() {
 		isNetworkErr := false
 		if config.DNS.FiltersUpdateIntervalHours != 0 && atomic.CompareAndSwapUint32(&refreshStatus, 0, 1) {
 			refreshLock.Lock()
-			_, isNetworkErr = refreshFiltersIfNecessary(false)
+			_, isNetworkErr = refreshFiltersIfNecessary(FilterRefreshBlocklists | FilterRefreshAllowlists)
 			refreshLock.Unlock()
 			refreshStatus = 0
 			if !isNetworkErr {
@@ -257,45 +257,33 @@ func periodicallyRefreshFilters() {
 }
 
 // Refresh filters
-func refreshFilters() (int, error) {
+// important:
-	if !atomic.CompareAndSwapUint32(&refreshStatus, 0, 1) {
+//  TRUE: ignore the fact that we're currently updating the filters
+func refreshFilters(whitelist bool, important bool) (int, error) {
+	set := atomic.CompareAndSwapUint32(&refreshStatus, 0, 1)
+	if !important && !set {
 		return 0, fmt.Errorf("Filters update procedure is already running")
 	}
 
 	refreshLock.Lock()
-	nUpdated, _ := refreshFiltersIfNecessary(true)
+	flags := FilterRefreshBlocklists
+	if whitelist {
+		flags = FilterRefreshAllowlists
+	}
+	nUpdated, _ := refreshFiltersIfNecessary(flags | FilterRefreshForce)
 	refreshLock.Unlock()
 	refreshStatus = 0
 	return nUpdated, nil
 }
 
-// Checks filters updates if necessary
+func refreshFiltersArray(filters *[]filter, force bool) (int, []filter, []bool, bool) {
-// If force is true, it ignores the filter.LastUpdated field value
-//
-// Algorithm:
-// . Get the list of filters to be updated
-// . For each filter run the download and checksum check operation
-// . For each filter:
-//  . If filter data hasn't changed, just set new update time on file
-//  . If filter data has changed:
-//    . rename the old file (1.txt -> 1.txt.old)
-//    . store the new data on disk (1.txt)
-//  . Pass new filters to dnsfilter object - it analyzes new data while the old filters are still active
-//  . dnsfilter activates new filters
-//  . Remove the old filter files (1.txt.old)
-//
-// Return the number of updated filters
-// Return TRUE - there was a network error and nothing could be updated
-func refreshFiltersIfNecessary(force bool) (int, bool) {
 	var updateFilters []filter
 	var updateFlags []bool // 'true' if filter data has changed
 
-	log.Debug("Filters: updating...")
-
 	now := time.Now()
 	config.RLock()
-	for i := range config.Filters {
+	for i := range *filters {
-		f := &config.Filters[i] // otherwise we will be operating on a copy
+		f := &(*filters)[i] // otherwise we will be operating on a copy
 
 		if !f.Enabled {
 			continue
@@ -316,7 +304,7 @@ func refreshFiltersIfNecessary(force bool) (int, bool) {
 	config.RUnlock()
 
 	if len(updateFilters) == 0 {
-		return 0, false
+		return 0, nil, nil, false
 	}
 
 	nfail := 0
@@ -333,7 +321,7 @@ func refreshFiltersIfNecessary(force bool) (int, bool) {
 	}
 
 	if nfail == len(updateFilters) {
-		return 0, true
+		return 0, nil, nil, true
 	}
 
 	updateCount := 0
@@ -354,8 +342,8 @@ func refreshFiltersIfNecessary(force bool) (int, bool) {
 		}
 
 		config.Lock()
-		for k := range config.Filters {
+		for k := range *filters {
-			f := &config.Filters[k]
+			f := &(*filters)[k]
 			if f.ID != uf.ID || f.URL != uf.URL {
 				continue
 			}
@@ -375,6 +363,61 @@ func refreshFiltersIfNecessary(force bool) (int, bool) {
 		config.Unlock()
 	}
 
+	return updateCount, updateFilters, updateFlags, false
+}
+
+const (
+	FilterRefreshForce      = 1 // ignore last file modification date
+	FilterRefreshAllowlists = 2 // update allow-lists
+	FilterRefreshBlocklists = 4 // update block-lists
+)
+
+// Checks filters updates if necessary
+// If force is true, it ignores the filter.LastUpdated field value
+// flags: FilterRefresh*
+//
+// Algorithm:
+// . Get the list of filters to be updated
+// . For each filter run the download and checksum check operation
+// . For each filter:
+//  . If filter data hasn't changed, just set new update time on file
+//  . If filter data has changed:
+//    . rename the old file (1.txt -> 1.txt.old)
+//    . store the new data on disk (1.txt)
+//  . Pass new filters to dnsfilter object - it analyzes new data while the old filters are still active
+//  . dnsfilter activates new filters
+//  . Remove the old filter files (1.txt.old)
+//
+// Return the number of updated filters
+// Return TRUE - there was a network error and nothing could be updated
+func refreshFiltersIfNecessary(flags int) (int, bool) {
+	log.Debug("Filters: updating...")
+
+	updateCount := 0
+	var updateFilters []filter
+	var updateFlags []bool
+	netError := false
+	netErrorW := false
+	force := false
+	if (flags & FilterRefreshForce) != 0 {
+		force = true
+	}
+	if (flags & FilterRefreshBlocklists) != 0 {
+		updateCount, updateFilters, updateFlags, netError = refreshFiltersArray(&config.Filters, force)
+	}
+	if (flags & FilterRefreshAllowlists) != 0 {
+		updateCountW := 0
+		var updateFiltersW []filter
+		var updateFlagsW []bool
+		updateCountW, updateFiltersW, updateFlagsW, netErrorW = refreshFiltersArray(&config.WhitelistFilters, force)
+		updateCount += updateCountW
+		updateFilters = append(updateFilters, updateFiltersW...)
+		updateFlags = append(updateFlags, updateFlagsW...)
+	}
+	if netError && netErrorW {
+		return 0, true
+	}
+
 	if updateCount != 0 {
 		enableFilters(false)
 

openapi/CHANGELOG.md

@@ -1,6 +1,21 @@
 # AdGuard Home API Change Log
 
 
+## v0.101: API changes
+
+### API: Refresh filters: POST /control/filtering/refresh
+
+* Added "whitelist" boolean parameter
+
+Request:
+
+	POST /control/filtering/refresh
+
+	{
+		"whitelist": true
+	}
+
+
 ## v0.100: API changes
 
 ### API: Get list of clients: GET /control/clients

openapi/openapi.yaml

@@ -570,6 +570,13 @@ paths:
                 This should work as intended, a `force` parameter is offered as last-resort attempt to make filter lists fresh.
 
                 If you ever find yourself using `force` to make something work that otherwise wont, this is a bug and report it accordingly.
+            consumes:
+            - application/json
+            parameters:
+              - in: "body"
+                name: "body"
+                schema:
+                  $ref: "#/definitions/FilterRefreshRequest"
             responses:
                 200:
                     description: OK
@@ -1196,6 +1203,13 @@ definitions:
             enabled:
                 type: "boolean"
 
+    FilterRefreshRequest:
+        type: "object"
+        description: "Refresh Filters request data"
+        properties:
+            whitelist:
+                type: "boolean"
+
     FilterCheckHostResponse:
         type: "object"
         description: "Check Host Result"