yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-02 11:49:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

add a few new hdfs directory patterns

Browse Source
This commit is contained in:
Eric Sun 2016-08-03 16:16:58 -07:00
parent 1cd5872369
commit 53d40c8392
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-dataset-extractor-standalone/src/main/java/wherehows/SchemaFetch.java
View File

@ -126,10 +126,10 @@ public class SchemaFetch {
if (!fstat.isDirectory()) {
// file
fileCount++;
} else if (objName.matches("(_|\\.|tmp|temp|_distcp|\\*|test).*")) {
} else if (objName.matches("(_|\\.|tmp|temp|_distcp|backup|\\*|test|trash).*")) {
// hidden/temporary fs object
hiddenFileCount++;
} else if (objName.matches("daily|hourly|monthly|weekly|year=[0-9]+|month=[0-9]+|country=.*")) {
} else if (objName.matches("daily|hourly|hourly.deduped|monthly|weekly|(ds|dt|datepartition|year|month|date)=[0-9-]+")) {
// temporal partition type
datePartitionCount++;
} else if (objName.matches(
@ -167,8 +167,8 @@ public class SchemaFetch {
throws IOException, InterruptedException, SQLException {
String curPath = path.toUri().getPath();
Path n = path;
//if (path.getName().matches("^(\\.|_|tmp|temp|test|\\*|archive|ARCHIVE|storkinternal).*"))
// return;
if (path.getName().matches("^(\\.|_|tmp|temp|test|trash|backup|archive|ARCHIVE|storkinternal).*"))
return;
logger.info(" -- scanPath(" + curPath + ")\n");
int x = isTable(path, scanFs);