Add verbose logging in MDG filtering (#1576)

vmutafov · web-flow · commit 831365d8a069 · 2020-02-10T11:04:55.000+02:00
* Add verbose printing in MDG

* Rename flag reading function in MDG

* Add 'verbose' prefix
diff --git a/test-app/app/build.gradle b/test-app/app/build.gradle
@@ -84,6 +84,7 @@ def computeBuildToolsVersion = { ->
 }
 
 def enableAnalytics = (project.hasProperty("gatherAnalyticsData") && project.gatherAnalyticsData == "true")
+def enableVerboseMDG = project.gradle.startParameter.logLevel.name() == 'DEBUG'
 def analyticsFilePath = "$rootDir/analytics/build-statistics.json"
 def analyticsCollector = project.ext.AnalyticsCollector.withOutputPath(analyticsFilePath)
 if (enableAnalytics) {
@@ -823,6 +824,10 @@ task buildMetadata(type: BuildToolTask) {
             paramz.add("analyticsFilePath=$analyticsFilePath")
         }
 
+        if(enableVerboseMDG){
+            paramz.add("verbose")
+        }
+
         args paramz.toArray()
     }
 }
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Builder.java b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Builder.java
@@ -67,9 +67,7 @@ static TreeNode build(List<String> paths) throws Exception {
         for (String className : classNames) {
             try {
                 SecuredNativeClassDescriptor clazz = SecuredClassRepository.INSTANCE.findClass(className);
-                if (!clazz.isUsageAllowed()) {
-                    throwMetadataSecurityViolationException(className);
-                } else {
+                if (clazz.isUsageAllowed()) {
                     tryCollectKotlinExtensionFunctions(clazz.getNativeDescriptor());
                 }
             } catch (Throwable e) {
@@ -90,9 +88,7 @@ static TreeNode build(List<String> paths) throws Exception {
                 // Class<?> clazz = Class.forName(className, false, loader);
 
                 SecuredNativeClassDescriptor clazz = SecuredClassRepository.INSTANCE.findClass(className);
-                if (!clazz.isUsageAllowed()) {
-                    throwMetadataSecurityViolationException(className);
-                } else {
+                if (clazz.isUsageAllowed()) {
                     generate(clazz.getNativeDescriptor(), root);
                 }
             } catch (Throwable e) {
@@ -106,10 +102,6 @@ static TreeNode build(List<String> paths) throws Exception {
         return root;
     }
 
-    private static void throwMetadataSecurityViolationException(String className){
-        throw new MetadataSecurityViolationException("Class " + className + " could not be used!");
-    }
-
     private static void tryCollectKotlinExtensionFunctions(NativeClassDescriptor classDescriptor) {
         if (classDescriptor instanceof KotlinClassDescriptor) {
             ExtensionFunctionsCollector extensionFunctionsCollector = new BytecodeExtensionFunctionsCollector(new BytecodeClassMetadataParser());
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Generator.java b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Generator.java
@@ -2,6 +2,7 @@
 
 import com.telerik.metadata.analytics.AnalyticsConfiguration;
 import com.telerik.metadata.security.filtering.input.user.UserPatternsCollection;
+import com.telerik.metadata.security.logging.console.MetadataFilterConsoleLogger;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -15,6 +16,7 @@
 
 public class Generator {
 
+    private static final String VERBOSE_FLAG_NAME = "verbose";
     private static final String ANALYTICS_ARGUMENT_BEGINNING = "analyticsFilePath=";
     private static final String MDG_OUTPUT_DIR = "mdg-output-dir.txt";
     private static final String MDG_JAVA_DEPENDENCIES = "mdg-java-dependencies.txt";
@@ -25,7 +27,7 @@ public class Generator {
      * @param args
      */
     public static void main(String[] args) {
-        enableAnalyticsBasedOnArgs(args);
+        enableFlaggedFeatures(args);
         UserPatternsCollection.INSTANCE.populateWhitelistEntriesFromFile(MDG_WHITELIST);
         UserPatternsCollection.INSTANCE.populateBlacklistEntriesFromFile(MDG_BLACKLIST);
 
@@ -63,17 +65,19 @@ public static void main(String[] args) {
         }
     }
 
-    private static void enableAnalyticsBasedOnArgs(String[] args){
+    private static void enableFlaggedFeatures(String[] args) {
         for (String arg : args) {
             if (arg.startsWith(ANALYTICS_ARGUMENT_BEGINNING)) {
                 String filePath = arg.replace(ANALYTICS_ARGUMENT_BEGINNING, "");
                 AnalyticsConfiguration.enableAnalytics(filePath);
+            } else if (VERBOSE_FLAG_NAME.equals(arg)) {
+                MetadataFilterConsoleLogger.INSTANCE.setEnabled(true);
             }
         }
     }
 
     public static List<String> getFileRows(String filename) throws IOException {
-        List<String> rows = new ArrayList<String>();
+        List<String> rows = new ArrayList<>();
         BufferedReader br = null;
         try {
             br = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/classes/SecuredClassRepository.kt b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/classes/SecuredClassRepository.kt
@@ -6,31 +6,32 @@ import com.telerik.metadata.security.filtering.blacklisting.MetadataBlacklist
 import com.telerik.metadata.security.filtering.input.user.UserPatternsCollection
 import com.telerik.metadata.security.filtering.matching.impl.PatternMatcherImpl
 import com.telerik.metadata.security.filtering.whitelisting.MetadataWhitelist
+import com.telerik.metadata.security.logging.console.MetadataFilterConsoleLogger
 import java.util.*
+import kotlin.collections.HashSet
 
 object SecuredClassRepository {
     private val cachedProviders = ArrayList<ClassMapProvider>()
     private val patternMatcher = PatternMatcherImpl()
     private val whitelist = MetadataWhitelist(UserPatternsCollection.whitelistEntries, patternMatcher)
     private val blacklist = MetadataBlacklist(UserPatternsCollection.blacklistEntries, patternMatcher)
+    private val alreadyLoggedClasses = HashSet<Pair<String, String>>()
 
     fun addToCache(classMapProvider: ClassMapProvider) {
         cachedProviders.add(classMapProvider)
     }
 
+    private fun NativeClassDescriptor.getSimplifiedClassName() = this.className.substring(this.packageName.length).replace('$', '.').trimStart('.')
+
     fun findClass(className: String): SecuredNativeClassDescriptor {
         val clazz: NativeClassDescriptor? = findClassFromProviders(className)
 
         if (clazz != null) {
-            val simplifiedClassName = clazz.className.substring(clazz.packageName.length).replace('$', '.').trimStart('.')
-
-            val isAllowedByWhitelist = whitelist.isAllowed(clazz.packageName, simplifiedClassName)
-            if (isAllowedByWhitelist) {
+            val simplifiedClassName = clazz.getSimplifiedClassName()
 
-                val isAllowedByBlacklist = blacklist.isAllowed(clazz.packageName, simplifiedClassName)
-                if (isAllowedByBlacklist) {
-                    return SecuredNativeClassDescriptor(true, clazz)
-                }
+            val isUsageAllowed = isClassUsageAllowed(clazz.packageName, simplifiedClassName)
+            if (isUsageAllowed) {
+                return SecuredNativeClassDescriptor(true, clazz)
             }
         }
 
@@ -41,15 +42,11 @@ object SecuredClassRepository {
         var clazz: NativeClassDescriptor? = findClassFromProviders(className)
 
         while (clazz != null) {
-            val simplifiedClassName = clazz.className.substring(clazz.packageName.length).replace('$', '.').trimStart('.')
+            val simplifiedClassName = clazz.getSimplifiedClassName()
 
-            val isAllowedByWhitelist = whitelist.isAllowed(clazz.packageName, simplifiedClassName)
-            if (isAllowedByWhitelist) {
-
-                val isAllowedByBlacklist = blacklist.isAllowed(clazz.packageName, simplifiedClassName)
-                if (isAllowedByBlacklist) {
-                    return SecuredNativeClassDescriptor(true, clazz)
-                }
+            val isUsageAllowed = isClassUsageAllowed(clazz.packageName, simplifiedClassName)
+            if (isUsageAllowed) {
+                return SecuredNativeClassDescriptor(true, clazz)
             }
 
             clazz = findClassFromProviders(clazz.superclassName)
@@ -58,6 +55,29 @@ object SecuredClassRepository {
         return SecuredNativeClassDescriptor(false, NativeClassDescriptor.Missing)
     }
 
+    private fun isClassUsageAllowed(packageName: String, className: String): Boolean {
+        val whitelistFilterResult = whitelist.isAllowed(packageName, className)
+        val blacklistFilterResult = blacklist.isAllowed(packageName, className)
+
+        val packageAndClassNamePair = packageName to className
+
+        if (whitelistFilterResult.isAllowed && blacklistFilterResult.isAllowed) {
+            if (packageAndClassNamePair !in alreadyLoggedClasses) {
+                MetadataFilterConsoleLogger.logAllowed(packageName, className, whitelistFilterResult, blacklistFilterResult)
+                alreadyLoggedClasses.add(packageAndClassNamePair)
+            }
+
+            return true
+        }
+
+        if (packageAndClassNamePair !in alreadyLoggedClasses) {
+            MetadataFilterConsoleLogger.logDisallowed(packageName, className, whitelistFilterResult, blacklistFilterResult)
+            alreadyLoggedClasses.add(packageAndClassNamePair)
+        }
+
+        return false
+    }
+
     private fun findClassFromProviders(className: String): NativeClassDescriptor? {
         var clazz: NativeClassDescriptor? = null
         for (classMapProvider in cachedProviders) {
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/MetadataFilter.kt b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/MetadataFilter.kt
@@ -1,5 +1,5 @@
 package com.telerik.metadata.security.filtering
 
 interface MetadataFilter {
-    fun isAllowed(packageName: String, className: String): Boolean
+    fun isAllowed(packageName: String, className: String): MetadataFilterResult
 }
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/MetadataFilterResult.kt b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/MetadataFilterResult.kt
@@ -0,0 +1,6 @@
+package com.telerik.metadata.security.filtering
+
+import com.telerik.metadata.security.filtering.input.PatternEntry
+import java.util.*
+
+data class MetadataFilterResult(val isAllowed: Boolean, val matchedPatternEntry: Optional<PatternEntry> = Optional.empty())
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/blacklisting/MetadataBlacklist.kt b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/blacklisting/MetadataBlacklist.kt
@@ -1,13 +1,19 @@
 package com.telerik.metadata.security.filtering.blacklisting
 
 import com.telerik.metadata.security.filtering.MetadataFilter
+import com.telerik.metadata.security.filtering.MetadataFilterResult
 import com.telerik.metadata.security.filtering.input.PatternEntry
 import com.telerik.metadata.security.filtering.matching.PatternMatcher
+import java.util.*
 
 class MetadataBlacklist(private val userInput: Collection<PatternEntry>, private val patternMatcher: PatternMatcher) : MetadataFilter {
 
-    override fun isAllowed(packageName: String, className: String) = !userInput.any { inputEntry ->
-        (inputEntry.packagePattern.isEmpty() || patternMatcher.match(inputEntry.packagePattern, packageName))
-                && (inputEntry.classPattern.isEmpty() || patternMatcher.match(inputEntry.classPattern, className))
+    override fun isAllowed(packageName: String, className: String): MetadataFilterResult {
+        val responsiblePatternEntry = userInput.firstOrNull { inputEntry ->
+            (inputEntry.packagePattern.isEmpty() || patternMatcher.match(inputEntry.packagePattern, packageName))
+                    && (inputEntry.classPattern.isEmpty() || patternMatcher.match(inputEntry.classPattern, className))
+        }
+
+        return if (responsiblePatternEntry != null) MetadataFilterResult(false, Optional.of(responsiblePatternEntry)) else MetadataFilterResult(true)
     }
 }
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/input/PatternEntry.kt b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/input/PatternEntry.kt
@@ -1,3 +1,11 @@
 package com.telerik.metadata.security.filtering.input
 
-data class PatternEntry(val packagePattern: String, val classPattern: String)
+data class PatternEntry(val packagePattern: String, val classPattern: String) {
+    companion object {
+        fun empty(): PatternEntry {
+            return PatternEntry("", "")
+        }
+    }
+
+    override fun toString() = "$packagePattern:$classPattern"
+}
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/whitelisting/MetadataWhitelist.kt b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/filtering/whitelisting/MetadataWhitelist.kt
@@ -1,15 +1,20 @@
 package com.telerik.metadata.security.filtering.whitelisting
 
 import com.telerik.metadata.security.filtering.MetadataFilter
+import com.telerik.metadata.security.filtering.MetadataFilterResult
 import com.telerik.metadata.security.filtering.input.PatternEntry
 import com.telerik.metadata.security.filtering.matching.PatternMatcher
+import java.util.*
 
 class MetadataWhitelist(private val userInput: Collection<PatternEntry>, private val patternMatcher: PatternMatcher) : MetadataFilter {
 
-    override fun isAllowed(packageName: String, className: String) = userInput.any { inputEntry ->
-        (inputEntry.packagePattern.isEmpty() || patternMatcher.match(inputEntry.packagePattern, packageName))
-                && (inputEntry.classPattern.isEmpty() || patternMatcher.match(inputEntry.classPattern, className))
-    }
+    override fun isAllowed(packageName: String, className: String): MetadataFilterResult {
+        val responsiblePatternEntry = userInput.firstOrNull { inputEntry ->
+            (inputEntry.packagePattern.isEmpty() || patternMatcher.match(inputEntry.packagePattern, packageName))
+                    && (inputEntry.classPattern.isEmpty() || patternMatcher.match(inputEntry.classPattern, className))
+        }
 
+        return if (responsiblePatternEntry != null) MetadataFilterResult(true, Optional.of(responsiblePatternEntry)) else MetadataFilterResult(false)
+    }
 
 }
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/logging/MetadataFilterLogger.kt b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/logging/MetadataFilterLogger.kt
@@ -0,0 +1,8 @@
+package com.telerik.metadata.security.logging
+
+import com.telerik.metadata.security.filtering.MetadataFilterResult
+
+interface MetadataFilterLogger {
+    fun logAllowed(packageName: String, className: String, whitelistFilterResult: MetadataFilterResult, blacklistFilterResult: MetadataFilterResult)
+    fun logDisallowed(packageName: String, className: String, whitelistFilterResult: MetadataFilterResult, blacklistFilterResult: MetadataFilterResult)
+}
diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/logging/console/MetadataFilterConsoleLogger.kt b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/security/logging/console/MetadataFilterConsoleLogger.kt
@@ -0,0 +1,60 @@
+package com.telerik.metadata.security.logging.console
+
+import com.telerik.metadata.security.filtering.MetadataFilterResult
+import com.telerik.metadata.security.logging.MetadataFilterLogger
+
+object MetadataFilterConsoleLogger : MetadataFilterLogger {
+
+    var isEnabled = false
+
+    override fun logAllowed(packageName: String, className: String, whitelistFilterResult: MetadataFilterResult, blacklistFilterResult: MetadataFilterResult) = logAction("Included", packageName, className, whitelistFilterResult, blacklistFilterResult)
+
+    override fun logDisallowed(packageName: String, className: String, whitelistFilterResult: MetadataFilterResult, blacklistFilterResult: MetadataFilterResult) = logAction("Blacklisted", packageName, className, whitelistFilterResult, blacklistFilterResult)
+
+    private fun logAction(action: String, packageName: String, className: String, whitelistFilterResult: MetadataFilterResult, blacklistFilterResult: MetadataFilterResult) {
+        if (isEnabled) {
+            val logBuilder = StringBuilder()
+
+            logBuilder
+                    .append("verbose: ")
+                    .append(action)
+                    .append(" ")
+                    .append(className)
+                    .append(" from ")
+                    .append(packageName)
+
+
+            val hasWhitelistPatternEntry = whitelistFilterResult.matchedPatternEntry.isPresent
+            val hasBlacklistPatternEntry = blacklistFilterResult.matchedPatternEntry.isPresent
+
+            if (hasWhitelistPatternEntry || hasBlacklistPatternEntry) {
+                logBuilder.append(" (")
+
+                var hasWrittenWhitelistingLog = false
+
+                if (hasWhitelistPatternEntry && whitelistFilterResult.isAllowed) {
+                    logBuilder.append("enabled by ")
+                            .append(whitelistFilterResult.matchedPatternEntry.get())
+
+                    hasWrittenWhitelistingLog = true
+                }
+
+                if (hasBlacklistPatternEntry && !blacklistFilterResult.isAllowed) {
+                    if (hasWrittenWhitelistingLog) {
+                        logBuilder.append(", ")
+                    }
+
+                    logBuilder.append("disabled by ")
+                            .append(blacklistFilterResult.matchedPatternEntry.get())
+                }
+
+                logBuilder.append(")")
+
+
+            }
+
+            val log = logBuilder.toString()
+            println(log)
+        }
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -84,6 +84,7 @@ def computeBuildToolsVersion = { ->`
`84`	`84`	`}`
`85`	`85`
`86`	`86`	`def enableAnalytics = (project.hasProperty("gatherAnalyticsData") && project.gatherAnalyticsData == "true")`
	`87`	`+def enableVerboseMDG = project.gradle.startParameter.logLevel.name() == 'DEBUG'`
`87`	`88`	`def analyticsFilePath = "$rootDir/analytics/build-statistics.json"`
`88`	`89`	`def analyticsCollector = project.ext.AnalyticsCollector.withOutputPath(analyticsFilePath)`
`89`	`90`	`if (enableAnalytics) {`
`@@ -823,6 +824,10 @@ task buildMetadata(type: BuildToolTask) {`
`823`	`824`	`paramz.add("analyticsFilePath=$analyticsFilePath")`
`824`	`825`	`}`
`825`	`826`
	`827`	`+ if(enableVerboseMDG){`
	`828`	`+ paramz.add("verbose")`
	`829`	`+ }`
	`830`	`+`
`826`	`831`	`args paramz.toArray()`
`827`	`832`	`}`
`828`	`833`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`package com.telerik.metadata.security.filtering`
`2`	`2`
`3`	`3`	`interface MetadataFilter {`
`4`		`- fun isAllowed(packageName: String, className: String): Boolean`
	`4`	`+ fun isAllowed(packageName: String, className: String): MetadataFilterResult`
`5`	`5`	`}`