/**
 * Copyright 2015 LinkedIn Corp. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
package dao;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.*;
import java.text.SimpleDateFormat;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.apache.commons.lang3.StringUtils;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.PreparedStatementCreator;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.support.GeneratedKeyHolder;
import org.springframework.jdbc.support.KeyHolder;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionTemplate;
import play.Logger;
import play.Play;
import play.libs.Json;
import models.*;

public class DatasetsDAO extends AbstractMySQLOpenSourceDAO
{
	public final static String HDFS_BROWSER_URL_KEY = "dataset.hdfs_browser.link";

	private final static String SELECT_PAGED_DATASET  = "SELECT " +
			"d.id, d.name, d.urn, d.source, d.properties, d.schema, " +
			"GROUP_CONCAT(o.owner_id ORDER BY o.sort_id ASC SEPARATOR ',') as owner_id, " +
			"GROUP_CONCAT(IFNULL(u.display_name, '*') ORDER BY o.sort_id ASC SEPARATOR ',') as owner_name, " +
			"FROM_UNIXTIME(source_created_time) as created, d.source_modified_time, " +
			"FROM_UNIXTIME(source_modified_time) as modified " +
			"FROM ( SELECT * FROM dict_dataset ORDER BY urn LIMIT ?, ? ) d " +
			"LEFT JOIN dataset_owner o on (d.id = o.dataset_id and (o.is_deleted is null OR o.is_deleted != 'Y')) " +
			"LEFT JOIN dir_external_user_info u on (o.owner_id = u.user_id and u.app_id = 300) " +
			"GROUP BY d.id, d.name, d.urn, d.source, d.properties, d.schema, " +
			"created, d.source_modified_time, modified";

	private final static String SELECT_PAGED_DATASET_BY_CURRENT_USER  = "SELECT " +
			"d.id, d.name, d.urn, d.source, d.schema, d.properties, " +
			"f.dataset_id, w.id as watch_id, " +
			"GROUP_CONCAT(o.owner_id ORDER BY o.sort_id ASC SEPARATOR ',') as owner_id, " +
			"GROUP_CONCAT(IFNULL(u.display_name, '*') ORDER BY o.sort_id ASC SEPARATOR ',') as owner_name, " +
			"FROM_UNIXTIME(source_created_time) as created, d.source_modified_time, " +
			"FROM_UNIXTIME(source_modified_time) as modified " +
			"FROM ( SELECT * FROM dict_dataset ORDER BY urn LIMIT ?, ?) d LEFT JOIN favorites f ON (" +
			"d.id = f.dataset_id and f.user_id = ?) " +
			"LEFT JOIN watch w on (d.id = w.item_id and w.item_type = 'dataset' and w.user_id = ?) " +
			"LEFT JOIN dataset_owner o on (d.id = o.dataset_id and (o.is_deleted is null OR o.is_deleted != 'Y')) " +
			"LEFT JOIN dir_external_user_info u on (o.owner_id = u.user_id and u.app_id = 300) " +
			"GROUP BY d.id, d.name, d.urn, d.source, d.schema, d.properties, f.dataset_id, " +
			"watch_id, created, d.source_modified_time, modified";

	private final static String GET_PAGED_DATASET_COUNT  = "SELECT count(*) FROM dict_dataset";

	private final static String SELECT_PAGED_DATASET_BY_URN  = "SELECT " +
			"d.id, d.name, d.urn, d.source, d.properties, d.schema, " +
			"GROUP_CONCAT(o.owner_id ORDER BY o.sort_id ASC SEPARATOR ',') as owner_id, " +
			"GROUP_CONCAT(IFNULL(u.display_name, '*') ORDER BY o.sort_id ASC SEPARATOR ',') as owner_name, " +
			"FROM_UNIXTIME(source_created_time) as created, d.source_modified_time, " +
			"FROM_UNIXTIME(source_modified_time) as modified " +
			"FROM ( SELECT * FROM dict_dataset WHERE urn LIKE ? ORDER BY urn limit ?, ? ) d " +
			"LEFT JOIN dataset_owner o on (d.id = o.dataset_id and (o.is_deleted is null OR o.is_deleted != 'Y')) " +
			"LEFT JOIN dir_external_user_info u on (o.owner_id = u.user_id and u.app_id = 300) " +
			"GROUP BY d.id, d.name, d.urn, d.source, d.properties, d.schema, created, " +
			"d.source_modified_time, modified";

	private final static String SELECT_PAGED_DATASET_BY_URN_CURRENT_USER  = "SELECT " +
			"d.id, d.name, d.urn, d.source, d.schema, " +
			"GROUP_CONCAT(o.owner_id ORDER BY o.sort_id ASC SEPARATOR ',') as owner_id, " +
			"GROUP_CONCAT(IFNULL(u.display_name, '*') ORDER BY o.sort_id ASC SEPARATOR ',') as owner_name, " +
			"d.properties, f.dataset_id, w.id as watch_id, " +
			"FROM_UNIXTIME(source_created_time) as created, d.source_modified_time, " +
			"FROM_UNIXTIME(source_modified_time) as modified " +
			"FROM ( SELECT * FROM dict_dataset WHERE urn LIKE ?  ORDER BY urn LIMIT ?, ? ) d " +
			"LEFT JOIN favorites f ON (" +
			"d.id = f.dataset_id and f.user_id = ?) " +
			"LEFT JOIN watch w ON (d.id = w.item_id and w.item_type = 'dataset' and w.user_id = ?) " +
			"LEFT JOIN dataset_owner o on (d.id = o.dataset_id and (o.is_deleted is null OR o.is_deleted != 'Y')) " +
			"LEFT JOIN dir_external_user_info u on (o.owner_id = u.user_id and u.app_id = 300) " +
			"GROUP BY d.id, d.name, d.urn, d.source, d.schema, d.properties, f.dataset_id, " +
			"watch_id, created, d.source_modified_time, modified";

	private final static String GET_PAGED_DATASET_COUNT_BY_URN  = "SELECT count(*) FROM dict_dataset WHERE urn LIKE ?";

	private final static String CHECK_SCHEMA_HISTORY  = "SELECT COUNT(*) FROM dict_dataset_schema_history " +
			"WHERE dataset_id = ? ";

	private final static String GET_DATASET_BY_ID = "SELECT d.id, max(s.id) as schema_history_id, d.name, " +
			"d.urn, d.source, d.schema, GROUP_CONCAT(o.owner_id ORDER BY o.sort_id ASC SEPARATOR ',') as owner_id, " +
			"GROUP_CONCAT(IFNULL(u.display_name, '*') ORDER BY o.sort_id ASC SEPARATOR ',') as owner_name, " +
			"GROUP_CONCAT(IFNULL(u.email, '*') ORDER BY o.sort_id ASC SEPARATOR ',') as owner_email, " +
			"FROM_UNIXTIME(source_created_time) as created, d.source_modified_time, " +
			"FROM_UNIXTIME(source_modified_time) as modified " +
			"FROM dict_dataset d LEFT JOIN dict_dataset_schema_history s on (d.id = s.dataset_id) " +
			"LEFT JOIN dataset_owner o on (d.id = o.dataset_id) " +
			"LEFT JOIN dir_external_user_info u on (o.owner_id = u.user_id) " +
			"WHERE d.id = ? GROUP BY d.id, d.name, d.urn, d.source, d.schema, " +
			"created, d.source_modified_time, modified";

	private final static String GET_DATASET_BY_ID_CURRENT_USER  = "SELECT DISTINCT d.id, " +
			"max(s.id) as schema_history_id, " +
			"d.name, d.urn, d.source, d.schema, " +
			"GROUP_CONCAT(o.owner_id ORDER BY o.sort_id ASC SEPARATOR ',') as owner_id, " +
			"GROUP_CONCAT(IFNULL(u.display_name, '*') ORDER BY o.sort_id ASC SEPARATOR ',') as owner_name, " +
			"GROUP_CONCAT(IFNULL(u.email, '*') ORDER BY o.sort_id ASC SEPARATOR ',') as owner_email, " +
			"FROM_UNIXTIME(d.source_created_time) as created, " +
			"d.source_modified_time, " +
			"FROM_UNIXTIME(d.source_modified_time) as modified, f.dataset_id, w.id as watch_id FROM dict_dataset d " +
			"LEFT JOIN favorites f ON (d.id = f.dataset_id and f.user_id = ?) " +
			"LEFT JOIN dict_dataset_schema_history s on (d.id = s.dataset_id) " +
			"LEFT JOIN watch w ON (w.item_id = d.id and w.item_type = 'dataset' and w.user_id = ?) " +
			"LEFT JOIN dataset_owner o on (d.id = o.dataset_id) " +
			"LEFT JOIN dir_external_user_info u on (o.owner_id = u.user_id) " +
			"WHERE d.id = ? GROUP BY d.id, d.name, d.urn, d.source, d.schema, created, " +
			"d.source_modified_time, modified, f.dataset_id, watch_id";

	private final static String GET_DATASET_COLUMNS_BY_DATASET_ID = "select dfd.field_id, dfd.sort_id, " +
			"dfd.parent_sort_id, dfd.parent_path, dfd.field_name, dfd.data_type, " +
			"dfd.is_nullable as nullable, dfd.is_indexed as indexed, dfd.is_partitioned as partitioned, " +
			"dfd.is_distributed as distributed, c.comment, " +
			"( SELECT count(*) FROM dict_dataset_field_comment ddfc " +
			"WHERE ddfc.dataset_id = dfd.dataset_id AND ddfc.field_id = dfd.field_id ) as comment_count " +
			"FROM dict_field_detail dfd LEFT JOIN dict_dataset_field_comment ddfc ON " +
			"(ddfc.field_id = dfd.field_id AND ddfc.is_default = true) LEFT JOIN field_comments c ON " +
			"c.id = ddfc.comment_id WHERE dfd.dataset_id = ? ORDER BY dfd.sort_id";

	private final static String GET_DATASET_COLUMNS_BY_DATASETID_AND_COLUMNID = "SELECT dfd.field_id, " +
			"dfd.sort_id, dfd.parent_sort_id, dfd.parent_path, dfd.field_name, dfd.data_type, " +
			"dfd.is_nullable as nullable, dfd.is_indexed as indexed, dfd.is_partitioned as partitioned, " +
			"dfd.is_distributed as distributed, c.text as comment, " +
			"( SELECT count(*) FROM dict_dataset_field_comment ddfc " +
			"WHERE ddfc.dataset_id = dfd.dataset_id AND ddfc.field_id = dfd.field_id ) as comment_count " +
			"FROM dict_field_detail dfd LEFT JOIN dict_dataset_field_comment ddfc ON " +
			"(ddfc.field_id = dfd.field_id AND ddfc.is_default = true) LEFT JOIN comments c ON " +
			"c.id = ddfc.comment_id WHERE dfd.dataset_id = ? AND dfd.field_id = ? ORDER BY dfd.sort_id";

	private final static String GET_DATASET_OWNERS_BY_ID = "SELECT o.owner_id, u.display_name, o.sort_id, " +
			"o.owner_type, o.namespace, o.owner_id_type, o.owner_source, o.owner_sub_type, o.confirmed_by, " +
			"u.email, u.is_active, is_group, o.modified_time " +
			"FROM dataset_owner o " +
			"LEFT JOIN dir_external_user_info u on (o.owner_id = u.user_id and u.app_id = 300) " +
			"WHERE o.dataset_id = ? and (o.is_deleted is null OR o.is_deleted != 'Y') ORDER BY o.sort_id";

	private final static String GET_DATASET_PROPERTIES_BY_DATASET_ID =
			"SELECT source, `properties` FROM dict_dataset WHERE id=?";

	private final static String GET_DATASET_SAMPLE_DATA_BY_ID =
			"SELECT dataset_id, urn, ref_id, data FROM dict_dataset_sample WHERE dataset_id=?";

	private final static String GET_DATASET_SAMPLE_DATA_BY_REFID =
			"SELECT data FROM dict_dataset_sample WHERE dataset_id=?";

	private final static String GET_DATASET_URN_BY_ID =
			"SELECT urn FROM dict_dataset WHERE id=?";

	private final static String GET_USER_ID = "select id FROM users WHERE username = ?";

	private final static String FAVORITE_A_DATASET =
			"INSERT INTO favorites (user_id, dataset_id, created) VALUES(?, ?, NOW())";

	private final static String UNFAVORITE_A_DATASET =
			"DELETE FROM favorites WHERE user_id = ? and dataset_id = ?";

	private final static String GET_OWNERSHIP_DATASETS =
			"SELECT SQL_CALC_FOUND_ROWS o.dataset_id, d.urn, "
					+ "GROUP_CONCAT(o.owner_id ORDER BY o.owner_id ASC SEPARATOR ',') as owner_id, "
					+ "GROUP_CONCAT(CASE WHEN o.confirmed_by > '' THEN o.owner_id ELSE null END "
					+ "ORDER BY o.owner_id ASC SEPARATOR ',') as confirmed_owner_id "
					+ "FROM dataset_owner o JOIN dict_dataset d ON o.dataset_id = d.id "
					+ "WHERE o.dataset_id in "
					+ "(select dataset_id from dataset_owner "
					+ "where owner_id = ? and (is_deleted != 'Y' or is_deleted is null)) "
					+ "GROUP BY o.dataset_id, d.urn ORDER BY d.urn LIMIT ?, ?";

	private final static String GET_OWNERSHIP_DATASETS_COUNT =
			"SELECT COUNT(*) FROM dataset_owner WHERE owner_id = ? and (is_deleted != 'Y' or is_deleted is null) ";

	private final static String GET_DATASET_OWNERS = "SELECT o.owner_id, o.namespace, " +
			"o.owner_type, o.owner_sub_type, " +
			"o.dataset_urn, u.display_name FROM dataset_owner o " +
			"LEFT JOIN dir_external_user_info u on (o.owner_id = u.user_id and u.app_id = 300) " +
			"WHERE dataset_id = ? and (o.is_deleted is null OR o.is_deleted != 'Y') ORDER BY sort_id";

	private final static String UPDATE_DATASET_OWNER_SORT_ID = "UPDATE dataset_owner " +
			"set sort_id = ? WHERE dataset_id = ? AND owner_id = ? AND namespace = ?";

	private final static String OWN_A_DATASET = "INSERT INTO dataset_owner (" +
			"dataset_id, owner_id, app_id, namespace, " +
			"owner_type, is_group, is_active, sort_id, created_time, modified_time, wh_etl_exec_id, dataset_urn) " +
			"VALUES(?, ?, 300, 'urn:li:corpuser', 'Producer', 'N', 'Y', 0, UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), 0, ?)";

	private final static String UNOWN_A_DATASET = "UPDATE dataset_owner " +
			"set is_deleted = 'Y' WHERE dataset_id = ? AND owner_id = ?  AND app_id = 300";

	private final static String UPDATE_DATASET_OWNERS = "INSERT INTO dataset_owner (dataset_id, owner_id, app_id, " +
			"namespace, owner_type, is_group, is_active, is_deleted, sort_id, created_time, " +
			"modified_time, wh_etl_exec_id, dataset_urn, owner_sub_type) " +
			"VALUES(?, ?, ?, ?, ?, ?, 'Y', 'N', ?, UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), 0, ?, ?) " +
			"ON DUPLICATE KEY UPDATE owner_type = ?, is_group = ?, is_deleted = 'N', " +
			"sort_id = ?, modified_time= UNIX_TIMESTAMP(), owner_sub_type=?";

	private final static String UPDATE_DATASET_CONFIRMED_OWNERS = "INSERT INTO dataset_owner " +
			"(dataset_id, owner_id, app_id, namespace, owner_type, is_group, is_active, " +
			"is_deleted, sort_id, created_time, modified_time, wh_etl_exec_id, dataset_urn, owner_sub_type, " +
			"owner_id_type, owner_source, confirmed_by, confirmed_on) " +
			"VALUES(?, ?, ?, ?, ?, ?, ?, 'N', ?, UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), 0, ?, ?, ?, ?, ?, ?) " +
			"ON DUPLICATE KEY UPDATE owner_type = ?, is_group = ?, is_deleted = 'N', " +
			"sort_id = ?, modified_time= UNIX_TIMESTAMP(), owner_sub_type=?, confirmed_by=?, confirmed_on=?";

	private final static String MARK_DATASET_OWNERS_AS_DELETED = "UPDATE dataset_owner " +
			"set is_deleted = 'Y' WHERE dataset_id = ?";

	private final static String GET_FAVORITES = "SELECT DISTINCT d.id, d.name, d.urn, d.source " +
			"FROM dict_dataset d JOIN favorites f ON d.id = f.dataset_id " +
			"JOIN users u ON f.dataset_id = d.id and f.user_id = u.id WHERE u.username = ? ORDER BY d.urn";

	private final static String GET_COMMENTS_BY_DATASET_ID = "SELECT SQL_CALC_FOUND_ROWS " +
			"c.id, c.dataset_id, c.text, c.created, c.modified, c.comment_type, " +
			"u.name, u.email, u.username FROM comments c JOIN users u ON c.user_id = u.id " +
			"WHERE c.dataset_id = ? ORDER BY modified DESC, id DESC LIMIT ?, ?";

	private final static String CREATE_DATASET_COMMENT = "INSERT INTO comments " +
			"(text, user_id, dataset_id, created, modified, comment_type) VALUES(?, ?, ?, NOW(), NOW(), ?)";

	private final static String GET_WATCHED_URN_ID = "SELECT id FROM watch " +
			"WHERE user_id = ? and item_type = 'urn' and urn = ?";

	private final static String GET_WATCHED_DATASET_ID = "SELECT id FROM watch " +
			"WHERE user_id = ? and item_id = ? and item_type = 'dataset'";

	private final static String WATCH_DATASET = "INSERT INTO watch " +
			"(user_id, item_id, urn, item_type, notification_type, created) VALUES(?, ?, NULL, 'dataset', ?, NOW())";

	private final static String UPDATE_DATASET_WATCH = "UPDATE watch " +
			"set user_id = ?, item_id = ?, notification_type = ? WHERE id = ?";

	private final static String WATCH_URN = "INSERT INTO watch " +
			"(user_id, item_id, urn, item_type, notification_type, created) VALUES(?, NULL, ?, 'urn', ?, NOW())";

	private final static String UPDATE_URN_WATCH = "update watch " +
			"set user_id = ?, urn = ?, notification_type = ? WHERE id = ?";

	private final static String CHECK_IF_COLUMN_COMMENT_EXIST = "SELECT id FROM field_comments " +
			"WHERE comment_crc32_checksum = CRC32(?) and comment = ?";

	private final static String CREATE_COLUMN_COMMENT = "INSERT INTO field_comments " +
			"(comment, user_id, created, modified, comment_crc32_checksum) VALUES(?, ?, NOW(), NOW(), CRC32(?))";

	private final static String UPDATE_DATASET_COMMENT = "UPDATE comments " +
			"SET text = ?, comment_type = ?, modified = NOW() WHERE id = ?";

	private final static String UPDATE_COLUMN_COMMENT = "UPDATE field_comments " +
			"SET comment = ?, modified = NOW() WHERE id = ?";

	private final static String DELETE_DATASET_COMMENT = "DELETE FROM comments WHERE id = ?";

	private final static String UNWATCH_DATASET = "DELETE FROM watch WHERE id = ?";

	private final static String GET_FIELD_COMMENT_BY_ID = "SELECT comment FROM dict_dataset_field_comment WHERE id = ?";

	private final static String GET_COLUMN_COMMENTS_BY_DATASETID_AND_COLUMNID = "SELECT SQL_CALC_FOUND_ROWS " +
			"c.id, u.name as author, " +
			"u.email as authorEmail, u.username as authorUsername, c.comment as `text`, " +
			"c.created, c.modified, dfc.field_id, dfc.is_default FROM dict_dataset_field_comment dfc " +
			"LEFT JOIN field_comments c ON dfc.comment_id = c.id LEFT JOIN users u ON c.user_id = u.id " +
			"WHERE dataset_id = ? AND field_id = ? ORDER BY is_default DESC, created LIMIT ?,?";

	private final static String CREATE_DATASET_COLUMN_COMMENT_REFERENCE =
			"INSERT IGNORE INTO dict_dataset_field_comment (dataset_id, field_id, comment_id) " +
			"VALUES (?,?,?)";

	private final static String CHECK_COLUMN_COMMENT_HAS_DEFAULT =
			"SELECT comment_id FROM dict_dataset_field_comment WHERE dataset_id = ? AND field_id = ? " +
			"AND is_default = True";

	private final static String SET_COLUMN_COMMENT_DEFAULT =
			"UPDATE dict_dataset_field_comment SET is_default = True " +
			"WHERE dataset_id = ? AND field_id = ? AND comment_id = ? " +
			"LIMIT 1";

	private final static String GET_COUNT_COLUMN_COMMENTS_BY_ID =
			"SELECT COUNT(*) FROM dict_dataset_field_comment WHERE dataset_id = ? and comment_id = ?";

	private final static String DELETE_COLUMN_COMMENT_AND_REFERENCE =
			"DELETE dfc, c FROM dict_dataset_field_comment dfc JOIN field_comments c " +
			"ON c.id = dfc.comment_id WHERE dfc.dataset_id = ? AND dfc.field_id = ? AND dfc.comment_id = ?";

	private final static String DELETE_COLUMN_COMMENT_REFERENCE =
			"DELETE FROM dict_dataset_field_comment WHERE dataset_id = ? AND column_id = ? " +
			"AND comment_id = ? LIMIT 1";

	private final static String GET_COLUMN_NAME_BY_ID =
			"SELECT UPPER(field_name) FROM dict_field_detail WHERE field_id = ?";

	private final static String GET_SIMILAR_COMMENTS_BY_FIELD_NAME =
			"SELECT count(*) as count, f.comment_id, c.comment FROM dict_field_detail d " +
			"JOIN dict_dataset_field_comment f on d.field_id = f.field_id and d.dataset_id = f.dataset_id " +
			"JOIN field_comments c on c.id = f.comment_id WHERE d.field_name = ? and f.is_default = 1 " +
			"GROUP BY f.comment_id, c.comment ORDER BY count DESC";

	private final static String SET_COLUMN_COMMENT_TO_FALSE = "UPDATE dict_dataset_field_comment " +
			"SET is_default = false WHERE dataset_id = ? AND field_id = ? AND is_default = true";

	private final static String INSERT_DATASET_COLUMN_COMMENT = "INSERT INTO " +
			"dict_dataset_field_comment (dataset_id, field_id, comment_id, is_default) " +
			"VALUES (?, ?, ?, true) ON DUPLICATE KEY UPDATE is_default = true";

	private final static String GET_SIMILAR_COLUMNS_BY_FIELD_NAME = "SELECT d.id as dataset_id, " +
			"d.name as dataset_name, dfd.field_id, dfd.data_type, fc.id as comment_id, fc.comment, d.source " +
			"FROM dict_field_detail dfd JOIN dict_dataset d ON dfd.dataset_id = d.id " +
			"LEFT JOIN dict_dataset_field_comment ddfc ON ddfc.dataset_id = d.id " +
			"AND ddfc.field_id = dfd.field_id AND ddfc.is_default = 1 " +
			"LEFT JOIN field_comments fc ON ddfc.comment_id = fc.id " +
			"WHERE dfd.dataset_id <> ? AND dfd.field_name = ? ORDER BY d.name asc";

	private final static String GET_DATASET_OWNER_TYPES = "SELECT DISTINCT owner_type " +
			"FROM dataset_owner WHERE owner_type is not null";

	private final static String GET_DATASET_DEPENDS_VIEW = "SELECT object_type, object_sub_type, " +
			"object_name, map_phrase, is_identical_map, mapped_object_dataset_id, " +
			"mapped_object_type,  mapped_object_sub_type, mapped_object_name " +
			"FROM cfg_object_name_map WHERE object_name = ?";

	private final static String GET_DATASET_REFERENCES = "SELECT object_type, object_sub_type, " +
			"object_name, object_dataset_id, map_phrase, is_identical_map, mapped_object_dataset_id, " +
			"mapped_object_type,  mapped_object_sub_type, mapped_object_name " +
			"FROM cfg_object_name_map WHERE mapped_object_name = ?";

	private final static String GET_DATASET_LISTVIEW_TOP_LEVEL_NODES = "SELECT DISTINCT " +
			"SUBSTRING_INDEX(urn, ':///', 1) as `name`, 0 as id, " +
			"LEFT(urn, INSTR(urn, ':///') + 3) as urn FROM dict_dataset order by 1";

	private final static String GET_DATASET_LISTVIEW_NODES_BY_URN = "SELECT DISTINCT " +
			"SUBSTRING_INDEX(SUBSTRING_INDEX(d.urn, ?, -1), '/', 1) as `name`, " +
			"concat(?, SUBSTRING_INDEX(SUBSTRING_INDEX(d.urn, ?, -1), '/', 1)) as urn, " +
			"s.id FROM dict_dataset d LEFT JOIN dict_dataset s " +
			"ON s.urn = concat(?, SUBSTRING_INDEX(SUBSTRING_INDEX(d.urn, ?, -1), '/', 1)) " +
			"WHERE d.urn LIKE ? ORDER BY 2";

	private final static String GET_DATASET_VERSIONS = "SELECT DISTINCT `version` " +
			"FROM dict_dataset_instance WHERE dataset_id = ? and `version` != '0' ORDER BY 1 DESC";

	private final static String GET_DATASET_NATIVE_NAME = "SELECT native_name " +
			"FROM dict_dataset_instance WHERE dataset_id = ? ORDER BY version_sort_id DESC limit 1";

	private final static String GET_DATASET_SCHEMA_TEXT_BY_VERSION = "SELECT schema_text " +
			"FROM dict_dataset_instance WHERE dataset_id = ? and version = ? ORDER BY db_id DESC limit 1";

	private final static String GET_DATASET_INSTANCES = "SELECT DISTINCT i.db_id, c.db_code FROM " +
			"dict_dataset_instance i JOIN cfg_database c ON i.db_id = c.db_id " +
			"WHERE i.dataset_id = ?";

	private final static String GET_DATASET_ACCESS_PARTITION_GAIN = "SELECT DISTINCT partition_grain " +
			"FROM log_dataset_instance_load_status WHERE dataset_id = ? order by 1";

	private final static String GET_DATASET_ACCESS_PARTITION_INSTANCES = "SELECT DISTINCT d.db_code " +
			"FROM log_dataset_instance_load_status l " +
			"JOIN cfg_database d on l.db_id = d.db_id WHERE dataset_id = ? and partition_grain = ? ORDER BY 1";

	private final static String GET_DATASET_ACCESS = "SELECT l.db_id, d.db_code, l.dataset_type, l.partition_expr, " +
			"l.data_time_expr, l.data_time_epoch, l.record_count, l.size_in_byte, l.log_time_epoch, " +
			"from_unixtime(l.log_time_epoch) as log_time_str FROM log_dataset_instance_load_status l " +
			"JOIN cfg_database d on l.db_id = d.db_id WHERE dataset_id = ? and partition_grain = ? " +
			"ORDER by l.data_time_expr DESC";

	private static final String GET_DATASET_COMPLIANCE_BY_DATASET_ID =
			"SELECT * FROM dataset_privacy_compliance WHERE dataset_id = ?";

	private static final String GET_DATASET_COMPLIANCE_BY_URN =
			"SELECT * FROM dataset_privacy_compliance WHERE dataset_urn = ?";

	private final static String INSERT_DATASET_COMPLIANCE =
			"INSERT INTO dataset_privacy_compliance (dataset_id, dataset_urn, compliance_purge_type, "
					+ "compliance_purge_entities, modified_time) VALUES (:id, :urn, :type, :entities, :modified) "
					+ "ON DUPLICATE KEY UPDATE compliance_purge_type = :type, compliance_purge_entities = :entities, "
					+ "modified_time = :modified";

	private static final String GET_DATASET_SECURITY_BY_DATASET_ID =
			"SELECT dataset_id, dataset_urn, dataset_classification, confidentiality, classification, record_owner_type, "
			+ "retention_policy, geographic_affinity, modified_time FROM dataset_security WHERE dataset_id = ?";

	private static final String GET_DATASET_SECURITY_BY_URN =
			"SELECT dataset_id, dataset_urn, dataset_classification, confidentiality, classification, record_owner_type, "
			+ "retention_policy, geographic_affinity, modified_time FROM dataset_security WHERE dataset_urn = ?";

	private final static String INSERT_DATASET_SECURITY =
			"INSERT INTO dataset_security (dataset_id, dataset_urn, dataset_classification, confidentiality, classification, "
					+ "record_owner_type, retention_policy, geographic_affinity, modified_time) "
					+ "VALUES (:id, :urn, :dataset_classification, :confidentiality, :classification, :ownerType, :policy, :geo, :modified) "
					+ "ON DUPLICATE KEY UPDATE "
					+ "dataset_classification = :dataset_classification, confidentiality = :confidentiality, "
					+ "classification = :classification, record_owner_type = :ownerType, retention_policy = :policy, "
					+ "geographic_affinity = :geo, modified_time = :modified";


	public static List<String> getDatasetOwnerTypes()
	{
		return getJdbcTemplate().queryForList(GET_DATASET_OWNER_TYPES, String.class);
	}

	public static ObjectNode getPagedDatasets(String urn, Integer page, Integer size, String user)
	{
		ObjectNode result = Json.newObject();

		Integer userId = UserDAO.getUserIDByUserName(user);

		javax.sql.DataSource ds = getJdbcTemplate().getDataSource();
		DataSourceTransactionManager tm = new DataSourceTransactionManager(ds);
		TransactionTemplate txTemplate = new TransactionTemplate(tm);
		final Integer id = userId;

		result = txTemplate.execute(new TransactionCallback<ObjectNode>() {
			public ObjectNode doInTransaction(TransactionStatus status) {

				ObjectNode resultNode = Json.newObject();
				List<Dataset> pagedDatasets = new ArrayList<Dataset>();
				List<Map<String, Object>> rows = null;
				if (id != null && id > 0)
				{
					if (StringUtils.isBlank(urn)) {
						rows = getJdbcTemplate().queryForList(
								SELECT_PAGED_DATASET_BY_CURRENT_USER,
								(page - 1) * size, size,
								id,
								id);
					} else {
						rows = getJdbcTemplate().queryForList(
								SELECT_PAGED_DATASET_BY_URN_CURRENT_USER,
								urn + "%",
								(page - 1) * size, size,
								id,
								id);
					}
				}
				else
				{
					if (StringUtils.isBlank(urn)) {
						rows = getJdbcTemplate().queryForList(
								SELECT_PAGED_DATASET,
								(page - 1) * size, size);
					} else {
						rows = getJdbcTemplate().queryForList(
								SELECT_PAGED_DATASET_BY_URN,
								urn + "%",
								(page - 1) * size, size);
					}

				}

				long count = 0;
				try {

					if (StringUtils.isBlank(urn)) {
						count = getJdbcTemplate().queryForObject(
								GET_PAGED_DATASET_COUNT,
								Long.class);
					}
					else
					{
						count = getJdbcTemplate().queryForObject(
								GET_PAGED_DATASET_COUNT_BY_URN,
								Long.class,
								urn + "%");
					}
				} catch (EmptyResultDataAccessException e) {
					Logger.error("Exception = " + e.getMessage());
				}

				for (Map row : rows) {

					Dataset ds = new Dataset();
					Timestamp modified = (Timestamp)row.get(DatasetWithUserRowMapper.DATASET_MODIFIED_TIME_COLUMN);
					ds.id = (Long)row.get(DatasetWithUserRowMapper.DATASET_ID_COLUMN);
					ds.name = (String)row.get(DatasetWithUserRowMapper.DATASET_NAME_COLUMN);
					ds.source = (String)row.get(DatasetWithUserRowMapper.DATASET_SOURCE_COLUMN);
					ds.urn = (String)row.get(DatasetWithUserRowMapper.DATASET_URN_COLUMN);
					ds.schema = (String)row.get(DatasetWithUserRowMapper.DATASET_SCHEMA_COLUMN);
					String strOwner = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN);
					String strOwnerName = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_NAME_COLUMN);
					Long sourceModifiedTime =
							(Long)row.get(DatasetWithUserRowMapper.DATASET_SOURCE_MODIFIED_TIME_COLUMN);
					String properties = (String)row.get(DatasetWithUserRowMapper.DATASET_PROPERTIES_COLUMN);
					try
					{
						if (StringUtils.isNotBlank(properties))
						{
							ds.properties = Json.parse(properties);
						}
					}
					catch (Exception e)
					{
						Logger.error(e.getMessage());
					}

					if (modified != null && sourceModifiedTime != null && sourceModifiedTime > 0)
					{
						ds.modified = modified;
						ds.formatedModified = modified.toString();
					}

					String[] owners = null;
					if (StringUtils.isNotBlank(strOwner))
					{
						owners = strOwner.split(",");
					}
					String[] ownerNames = null;
					if (StringUtils.isNotBlank(strOwnerName))
					{
						ownerNames = strOwnerName.split(",");
					}
					ds.owners = new ArrayList<User>();
					if (owners != null && ownerNames != null)
					{
						if (owners.length == ownerNames.length)
						{
							for (int i = 0; i < owners.length; i++)
							{
								User datasetOwner = new User();
								datasetOwner.userName = owners[i];
								if (datasetOwner.userName.equalsIgnoreCase(user))
								{
									ds.isOwned = true;
								}
								if (StringUtils.isBlank(ownerNames[i]) || ownerNames[i].equalsIgnoreCase("*"))
								{
									datasetOwner.name = owners[i];
								}
								else
								{
									datasetOwner.name = ownerNames[i];
								}
								ds.owners.add(datasetOwner);
							}
						}
						else
						{
							Logger.error("getPagedDatasets get wrong owner and names. Dataset ID: "
									+ Long.toString(ds.id) + " Owner: " + owners + " Owner names: " + ownerNames);
						}
					}

					Integer favoriteId = (Integer)row.get(DatasetWithUserRowMapper.FAVORITE_DATASET_ID_COLUMN);
					Long watchId = (Long)row.get(DatasetWithUserRowMapper.DATASET_WATCH_ID_COLUMN);

					Long schemaHistoryRecordCount = 0L;
					try
					{
						schemaHistoryRecordCount = getJdbcTemplate().queryForObject(
								CHECK_SCHEMA_HISTORY,
								Long.class,
								ds.id);
					}
					catch (EmptyResultDataAccessException e)
					{
						Logger.error("Exception = " + e.getMessage());
					}

					if (StringUtils.isNotBlank(ds.urn))
					{
						if (ds.urn.substring(0, 4).equalsIgnoreCase(DatasetRowMapper.HDFS_PREFIX))
						{
							ds.hdfsBrowserLink = Play.application().configuration().getString(HDFS_BROWSER_URL_KEY) +
									ds.urn.substring(DatasetRowMapper.HDFS_URN_PREFIX_LEN);
						}
					}
					if (favoriteId != null && favoriteId > 0)
					{
						ds.isFavorite = true;
					}
					else
					{
						ds.isFavorite = false;
					}
					if (watchId != null && watchId > 0)
					{
						ds.watchId = watchId;
						ds.isWatched = true;
					}
					else
					{
						ds.isWatched = false;
						ds.watchId = 0L;
					}
					if (schemaHistoryRecordCount != null && schemaHistoryRecordCount > 0)
					{
						ds.hasSchemaHistory = true;
					}
					else
					{
						ds.hasSchemaHistory = false;
					}
					pagedDatasets.add(ds);
				}

				resultNode.put("count", count);
				resultNode.put("page", page);
				resultNode.put("itemsPerPage", size);
				resultNode.put("totalPages", (int) Math.ceil(count / ((double) size)));
				resultNode.set("datasets", Json.toJson(pagedDatasets));
				return resultNode;
			}
		});
		return result;
	}

	private static long getPagedDatasetsOwnedBy(String userId, int page, int size, List<Map<String, Object>> rows) {
		long count = 0;
		if (StringUtils.isNotBlank(userId)) {
			rows.addAll(getJdbcTemplate().queryForList(GET_OWNERSHIP_DATASETS, userId, (page - 1) * size, size));

			try {
				//count = getJdbcTemplate().queryForObject("SELECT FOUND_ROWS()", Long.class);
				count = getJdbcTemplate().queryForObject(GET_OWNERSHIP_DATASETS_COUNT, new String[]{userId}, Long.class);
			} catch (EmptyResultDataAccessException e) {
				Logger.error("Exception = " + e.getMessage());
			}
		}
		return count;
	}

	public static ObjectNode getDatasetOwnedBy(String userId, int page, int size) {
		List<Map<String, Object>> rows = new ArrayList<>(size);
		long count = getPagedDatasetsOwnedBy(userId, page, size, rows);

		List<DashboardDataset> datasets = new ArrayList<>();
		for (Map row : rows) {
			DashboardDataset dataset = new DashboardDataset();
			dataset.datasetId = (Long) row.get("dataset_id");
			dataset.datasetName = (String) row.get("urn");
			dataset.ownerId = (String) row.get("owner_id");
			dataset.confirmedOwnerId = (String) row.get("confirmed_owner_id");
			datasets.add(dataset);
		}

		ObjectNode resultNode = Json.newObject()
				.put("status", "ok")
				.put("count", count)
				.put("page", page)
				.put("itemsPerPage", size)
				.put("totalPages", (int) Math.ceil(count / ((double) size)));
		resultNode.set("datasets", Json.toJson(datasets));
		return resultNode;
	}

	public static ObjectNode ownDataset(int id, String user)
	{
		ObjectNode resultNode = Json.newObject();
		boolean result = false;
		List<Map<String, Object>> rows = null;

		rows = getJdbcTemplate().queryForList(GET_DATASET_OWNERS, id);
		int sortId = 0;
		for (Map row : rows)
		{
			String ownerId = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN);
			String namespace = (String)row.get("namespace");
			int ret = getJdbcTemplate().update(UPDATE_DATASET_OWNER_SORT_ID, ++sortId, id, ownerId, namespace);
			if (ret <= 0)
			{
				Logger.warn("ownDataset update sort_id failed. Dataset id is : " +
						Long.toString(id) + " owner_id is : " + ownerId + " namespace is : " + namespace);
			}
		}

		String urn = getDatasetUrnById(id);
		int status = getJdbcTemplate().update(
				UPDATE_DATASET_OWNERS,
				id,
				user,
				300,
				"urn:li:corpuser",
				"Producer",
				"N",
				0,
				urn,
				"",
				"Producer",
				"N",
				0,
				"");
		if (status > 0)
		{
			result = true;
		}
		rows = getJdbcTemplate().queryForList(GET_DATASET_OWNERS, id);
		List<User> owners = new ArrayList<User>();
		for (Map row : rows)
		{
			String ownerId = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN);
			String dislayName = (String)row.get("display_name");
			if (StringUtils.isBlank(dislayName))
			{
				dislayName = ownerId;
			}
			User owner = new User();
			owner.userName = ownerId;
			owner.name = dislayName;
			owners.add(owner);
		}
		if (result)
		{
			resultNode.put("status", "success");
		}
		else
		{
			resultNode.put("status", "failed");
		}
		resultNode.set("owners", Json.toJson(owners));
		return resultNode;
	}

	public static ObjectNode unownDataset(int id, String user)
	{
		ObjectNode resultNode = Json.newObject();
		boolean result = false;
		int ret = getJdbcTemplate().update(UNOWN_A_DATASET, id, user);
		if (ret > 0)
		{
			result = true;
		}
		List<Map<String, Object>> rows = null;
		rows = getJdbcTemplate().queryForList(GET_DATASET_OWNERS, id);
		List<User> owners = new ArrayList<User>();
		int sortId = 0;
		for (Map row : rows)
		{
			String ownerId = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN);
			String dislayName = (String)row.get("display_name");
			String namespace = (String)row.get("namespace");
			if (StringUtils.isBlank(dislayName))
			{
				dislayName = ownerId;
			}
			User owner = new User();
			owner.userName = ownerId;
			owner.name = dislayName;
			owners.add(owner);
			int updatedRows = getJdbcTemplate().update(UPDATE_DATASET_OWNER_SORT_ID, sortId++, id, ownerId, namespace);
			if (ret <= 0)
			{
				Logger.warn("ownDataset update sort_id failed. Dataset id is : " +
						Long.toString(id) + " owner_id is : " + ownerId + " namespace is : " + namespace);
			}
		}
		if (result)
		{
			resultNode.put("status", "success");
		}
		else
		{
			resultNode.put("status", "failed");
		}
		resultNode.set("owners", Json.toJson(owners));
		return resultNode;
	}

	public static String getDatasetUrnById(int dataset_id) {
		try {
			return getJdbcTemplate().queryForObject(GET_DATASET_URN_BY_ID, String.class, dataset_id);
		} catch(EmptyResultDataAccessException e) {
			Logger.error("Can not find URN for dataset id: " + dataset_id + ", Exception: " + e.getMessage());
		}
		return null;
	}

	public static Dataset getDatasetByID(int id, String user)
	{
		Dataset dataset = null;
		Integer userId = 0;
		if (StringUtils.isNotBlank(user))
		{
			try
			{
				userId = getJdbcTemplate().queryForObject(GET_USER_ID, Integer.class,	user);
			}
			catch(EmptyResultDataAccessException e)
			{
				Logger.error("Dataset getDatasetByID get user id failed, username = " + user);
				Logger.error("Exception = " + e.getMessage());
			}
		}
		try
		{
			if (userId != null && userId > 0)
			{
				dataset = getJdbcTemplate().queryForObject(GET_DATASET_BY_ID_CURRENT_USER, new DatasetWithUserRowMapper(),
						userId,	userId, id);
			}
			else
			{
				dataset = getJdbcTemplate().queryForObject(GET_DATASET_BY_ID, new DatasetRowMapper(), id);
			}
		}
		catch(EmptyResultDataAccessException e)
		{
			Logger.error("Dataset getDatasetByID failed, id = " + id);
			Logger.error("Exception = " + e.getMessage());
		}

		return dataset;
	}

	public static List<DatasetColumn> getDatasetColumnByID(int datasetId, int columnId)
	{
		return getJdbcTemplate().query(GET_DATASET_COLUMNS_BY_DATASETID_AND_COLUMNID,
      new DatasetColumnRowMapper(), datasetId, columnId);
	}

	public static List<DatasetColumn> getDatasetColumnsByID(int datasetId)
	{
		return getJdbcTemplate().query(GET_DATASET_COLUMNS_BY_DATASET_ID,
      new DatasetColumnRowMapper(), datasetId);
	}

	public static JsonNode getDatasetPropertiesByID(int id)
	{
		String properties = "";
		String source = "";
		List<Map<String, Object>> rows = null;
		JsonNode propNode = null;

		rows = getJdbcTemplate().queryForList(GET_DATASET_PROPERTIES_BY_DATASET_ID, id);

		for (Map row : rows) {
			properties = (String)row.get("properties");
			source = (String)row.get("source");
			break;
		}

		if (StringUtils.isNotBlank(properties))
		{
			try {
				propNode = Json.parse(properties);

				if (propNode != null
						&& propNode.isContainerNode()
						&& propNode.has("url")
						&& StringUtils.isNotBlank(source)
						&& source.equalsIgnoreCase("pinot"))
				{
					URL url = new URL(propNode.get("url").asText());
					BufferedReader in =
							new BufferedReader(new InputStreamReader(url.openStream()));
					String resultString = "";
					String str;

					while ((str = in.readLine()) != null) {
						resultString += str;
					}

					in.close();
					JsonNode resultNode = Json.parse(resultString);

					if (resultNode == null)
					{
						return propNode;
					}
					else{
						return resultNode;
					}
				}
			}
			catch(Exception e) {
				Logger.error("Dataset getDatasetPropertiesByID parse properties failed, id = " + id);
				Logger.error("Exception = " + e.getMessage());
			}
		}

		return propNode;
	}

	public static JsonNode getDatasetSampleDataByID(int id)
	{
		List<Map<String, Object>> rows = null;
		JsonNode sampleNode = null;
		String strSampleData = null;
		Integer refID = 0;

		rows = getJdbcTemplate().queryForList(GET_DATASET_SAMPLE_DATA_BY_ID, id);

		for (Map row : rows) {
			refID = (Integer)row.get("ref_id");
			strSampleData = (String)row.get("data");
			break;
		}

		if (refID != null && refID != 0)
		{
			rows = null;
			rows = getJdbcTemplate().queryForList(GET_DATASET_SAMPLE_DATA_BY_REFID, refID);
			for (Map row : rows) {
				strSampleData = (String)row.get("data");
				break;
			}
		}

		if (StringUtils.isNotBlank(strSampleData)) {
			try {
				sampleNode = Json.parse(strSampleData);
				return utils.SampleData.secureSampleData(sampleNode);
			} catch (Exception e) {
				Logger.error("Dataset getDatasetSampleDataByID parse properties failed, id = " + id);
				Logger.error("Exception = " + e.getMessage());
			}
		}

		return sampleNode;
	}

	public static List<DatasetOwner> getDatasetOwnersByID(int id) {
		return getJdbcTemplate().query(GET_DATASET_OWNERS_BY_ID, new DatasetOwnerRowMapper(), id);
	}

	public static List<ImpactDataset> getImpactAnalysisByID(int id)
	{
		String urn = getDatasetUrnById(id);
		return LineageDAO.getImpactDatasetsByUrn(urn);
	}

	public static boolean favorite(int id, String user)
	{
		ObjectNode resultNode = Json.newObject();
		boolean result = false;
		Integer userId = UserDAO.getUserIDByUserName(user);

		if (userId != null && userId !=0)
		{
			int row = getJdbcTemplate().update(FAVORITE_A_DATASET, userId, id);
			if (row > 0)
			{
				result = true;
			}
		}
		return result;
	}

	public static boolean unfavorite(int id, String user)
	{
		ObjectNode resultNode = Json.newObject();
		boolean result = false;
		Integer userId = UserDAO.getUserIDByUserName(user);

		if (userId != null && userId !=0)
		{
			int row = getJdbcTemplate().update(UNFAVORITE_A_DATASET, userId, id);
			if (row > 0)
			{
				result = true;
			}
		}
		return result;
	}

	public static ObjectNode getFavorites(String user)
	{
		List<Dataset> favorites = new ArrayList<Dataset>();

		if (StringUtils.isNotBlank(user))
		{
			List<Map<String, Object>> rows = null;
			rows = getJdbcTemplate().queryForList(GET_FAVORITES, user);

			for (Map row : rows) {

				Dataset ds = new Dataset();
				ds.id = (long)row.get(DatasetRowMapper.DATASET_ID_COLUMN);
				ds.name = (String)row.get(DatasetRowMapper.DATASET_NAME_COLUMN);
				ds.source = (String)row.get(DatasetRowMapper.DATASET_SOURCE_COLUMN);
				ds.urn = (String)row.get(DatasetRowMapper.DATASET_URN_COLUMN);
				favorites.add(ds);
			}
		}

		ObjectNode result = Json.newObject();
		result.put("count", favorites.size());
		result.set("datasets", Json.toJson(favorites));

		return result;
	}

	public static ObjectNode getPagedDatasetComments(String userName, int id, int page, int size)
	{
		ObjectNode result = Json.newObject();

		javax.sql.DataSource ds = getJdbcTemplate().getDataSource();
		DataSourceTransactionManager tm = new DataSourceTransactionManager(ds);
		TransactionTemplate txTemplate = new TransactionTemplate(tm);

		result = txTemplate.execute(new TransactionCallback<ObjectNode>() {
			public ObjectNode doInTransaction(TransactionStatus status) {

				List<DatasetComment> pagedComments = getJdbcTemplate().query(
							GET_COMMENTS_BY_DATASET_ID,
							new DatasetCommentRowMapper(),
							id,
							(page - 1) * size, size);

				long count = 0;
				try {
					count = getJdbcTemplate().queryForObject(
							"SELECT FOUND_ROWS()",
							Long.class);
				} catch (EmptyResultDataAccessException e) {
					Logger.error("Exception = " + e.getMessage());
				}

				if (pagedComments != null)
				{
					for(DatasetComment dc : pagedComments)
					{
						if(StringUtils.isNotBlank(userName) && userName.equalsIgnoreCase(dc.authorUserName))
						{
							dc.isAuthor = true;
						}
					}
				}

				ObjectNode resultNode = Json.newObject();
				resultNode.set("comments", Json.toJson(pagedComments));
				resultNode.put("count", count);
				resultNode.put("page", page);
				resultNode.put("itemsPerPage", size);
				resultNode.put("totalPages", (int) Math.ceil(count / ((double) size)));

				return resultNode;
			}
		});
		return result;
	}

	public static boolean postComment(int datasetId, Map<String, String[]> commentMap, String user)
	{
		boolean result = false;
		if ((commentMap == null) || commentMap.size() == 0)
		{
			return false;
		}

		String text = "";
		if (commentMap.containsKey("text")) {
			String[] textArray = commentMap.get("text");
			if (textArray != null && textArray.length > 0)
			{
				text = textArray[0];
			}
		}
		if (StringUtils.isBlank(text))
		{
      return false;
		}

		String type = "Comment";
		if (commentMap.containsKey("type")) {
			String[] typeArray = commentMap.get("type");
			if (typeArray != null && typeArray.length > 0)
			{
				type = typeArray[0];
			}
		}

		Integer commentId = 0;
		if (commentMap.containsKey("id")) {
			String[] idArray = commentMap.get("id");
			if (idArray != null && idArray.length > 0)
			{
				String idStr = idArray[0];
				try
				{
					commentId = Integer.parseInt(idStr);
				}
				catch(NumberFormatException e)
				{
					Logger.error("DatasetDAO postComment wrong id parameter. Error message: " +
							e.getMessage());
					commentId = 0;
				}
			}
		}

		Integer userId = UserDAO.getUserIDByUserName(user);

		if (userId != null && userId !=0)
		{
			if (commentId != null && commentId != 0)
			{
				int row = getJdbcTemplate().update(UPDATE_DATASET_COMMENT, text, type, commentId);
				if (row > 0)
				{
					result = true;
				}
			}
			else
			{
				int row = getJdbcTemplate().update(CREATE_DATASET_COMMENT, text, userId, datasetId, type);
				if (row > 0)
				{
					result = true;
				}
			}
		}
		return result;
	}

	public static boolean deleteComment(int id)
	{
		boolean result = false;
		int row = getJdbcTemplate().update(DELETE_DATASET_COMMENT, id);
		if (row > 0)
		{
			result = true;
		}
		return result;
	}

	public static Long getWatchId(String urn, String user)
	{
		Long id = 0L;
		Integer userId = UserDAO.getUserIDByUserName(user);

		if (userId != null && userId !=0)
		{
			List<Map<String, Object>> rows = null;
			rows = getJdbcTemplate().queryForList(GET_WATCHED_URN_ID, userId, urn);
			if (rows != null)
			{
				for (Map row : rows) {
					id = (Long)row.get("id");
					break;
				}
			}
		}
		return id;
	}

	public static String watchDataset(int datasetId, Map<String, String[]> params, String user)
	{
		String message = "Internal error";
		if (params == null || params.size() == 0)
		{
			return "Empty post body";
		}

		String notificationType = "";
		if (params.containsKey("notification_type")) {
			String[] notificationTypeArray = params.get("notification_type");
			if (notificationTypeArray != null && notificationTypeArray.length > 0)
			{
				notificationType = notificationTypeArray[0];
			}
		}
		if (StringUtils.isBlank(notificationType))
		{
			return "notification_type is missing";
		}

		Long watchId = 0L;
		if (params.containsKey("id")) {
			String[] watchIdArray = params.get("id");
			if (watchIdArray != null && watchIdArray.length > 0)
			{
				try
				{
					watchId = Long.parseLong(watchIdArray[0]);
				}
				catch(NumberFormatException e)
				{
					Logger.error("DatasetDAO watchDataset wrong watch_id parameter. Error message: " +
							e.getMessage());
					watchId = 0L;
				}
			}
		}

		Integer userId = UserDAO.getUserIDByUserName(user);

		if (userId != null && userId !=0)
		{
			List<Map<String, Object>> rows = null;
			rows = getJdbcTemplate().queryForList(GET_WATCHED_DATASET_ID, userId, datasetId);
			if (rows != null && rows.size() > 0)
			{
				message = "watch item already exist";
			}
			else
			{
				int row = 0;
				if (watchId > 0)
				{
					row = getJdbcTemplate().update(UPDATE_DATASET_WATCH, userId, datasetId, notificationType, watchId);
				}
				else
				{
					row = getJdbcTemplate().update(WATCH_DATASET, userId, datasetId, notificationType);
				}
				if (row > 0)
				{
					message = "";
				}
			}
		}
		else
		{
			message = "User not found";
		}
		return message;
	}

	public static boolean unwatch(int id)
	{
		boolean result = false;
		int row = getJdbcTemplate().update(UNWATCH_DATASET, id);
		if (row > 0)
		{
			result = true;
		}
		return result;
	}

	public static String watchURN(Map<String, String[]> params, String user)
	{
		String message = "Internal error";
		if (params == null || params.size() == 0)
		{
			return "Empty post body";
		}

			String urn = "";
		if (params.containsKey("urn")) {
			String[] urnArray = params.get("urn");
			if (urnArray != null && urnArray.length > 0)
			{
				urn = urnArray[0];
			}
		}
		if (StringUtils.isBlank(urn))
		{
			return "urn parameter is missing";
		}

		String notificationType = "";
		if (params.containsKey("notification_type")) {
			String[] notificationTypeArray = params.get("notification_type");
			if (notificationTypeArray != null && notificationTypeArray.length > 0)
			{
				notificationType = notificationTypeArray[0];
			}
		}
		if (StringUtils.isBlank(notificationType))
		{
			return "notification_type is missing";
		}

		Long watchId = 0L;
		if (params.containsKey("id")) {
			String[] watchIdArray = params.get("id");
			if (watchIdArray != null && watchIdArray.length > 0)
			{
				try
				{
					watchId = Long.parseLong(watchIdArray[0]);
				}
				catch(NumberFormatException e)
				{
					Logger.error("DatasetDAO watchURN wrong watch_id parameter. Error message: " +
							e.getMessage());
					watchId = 0L;
				}
			}
		}

		Integer userId = UserDAO.getUserIDByUserName(user);

		if (userId != null && userId !=0)
		{
			List<Map<String, Object>> rows = null;
			rows = getJdbcTemplate().queryForList(GET_WATCHED_URN_ID, userId, urn);
			if (rows != null && rows.size() > 0)
			{
				message = "watch item is already exist";
			}
			else
			{
				int row = 0;
				if (watchId > 0)
				{
					row = getJdbcTemplate().update(UPDATE_URN_WATCH, userId, urn, notificationType, watchId);
				}
				else
				{
					row = getJdbcTemplate().update(WATCH_URN, userId, urn, notificationType);
				}
				if (row > 0)
				{
					message = "";
				}
			}
		}
		else
		{
			message = "User not found";
		}
		return message;
	}

	public static ObjectNode getPagedDatasetColumnComments(String userName, int datasetId, int columnId, int page, int size)
	{
		ObjectNode result = Json.newObject();

		javax.sql.DataSource ds = getJdbcTemplate().getDataSource();
		DataSourceTransactionManager tm = new DataSourceTransactionManager(ds);
		TransactionTemplate txTemplate = new TransactionTemplate(tm);

		result = txTemplate.execute(new TransactionCallback<ObjectNode>() {
			public ObjectNode doInTransaction(TransactionStatus status) {

				ObjectNode resultNode = Json.newObject();
				long count = 0;
				int start = (page - 1) * size;
				int end = start + size;
				List<DatasetColumnComment> pagedComments = new ArrayList<DatasetColumnComment>();
				List<Map<String, Object>> rows = null;

				rows = getJdbcTemplate().queryForList(
					GET_COLUMN_COMMENTS_BY_DATASETID_AND_COLUMNID,
					datasetId,
					columnId,
					start,
					end
				);
				for (Map row : rows) {
					Long id = (Long)row.get("id");
					String author = (String)row.get("author");
					String authorEmail = (String)row.get("authorEmail");
					String authorUsername = (String)row.get("authorUsername");
					String text = (String)row.get("text");
					String created = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss").format((Timestamp)row.get("created"));
					String modified = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss").format((Timestamp)row.get("modified"));
					Long columnId = (Long)row.get("field_id");
					boolean isDefault = (Boolean)row.get("is_default");

					DatasetColumnComment datasetColumnComment = new DatasetColumnComment();
					datasetColumnComment.id = id;
					datasetColumnComment.author = author;
					datasetColumnComment.authorEmail = authorEmail;
					datasetColumnComment.authorUsername = authorUsername;
					datasetColumnComment.text = text;
					datasetColumnComment.created = created;
					datasetColumnComment.modified = modified;
					datasetColumnComment.columnId = columnId;
					datasetColumnComment.isDefault = isDefault;
					pagedComments.add(datasetColumnComment);
				}

				try {
					count = getJdbcTemplate().queryForObject("SELECT FOUND_ROWS()", Long.class);
				}
				catch (EmptyResultDataAccessException e) {
					Logger.error("Exception = " + e.getMessage());
				}

				if (pagedComments != null)
				{
					for(DatasetColumnComment dc : pagedComments)
					{
						if(StringUtils.isNotBlank(userName) && userName.equalsIgnoreCase(dc.authorUsername))
						{
							dc.isAuthor = true;
						}
					}
				}

				resultNode.set("comments", Json.toJson(pagedComments));
				resultNode.put("count", count);
				resultNode.put("page", page);
				resultNode.put("itemsPerPage", size);
				resultNode.put("totalPages", (int) Math.ceil(count / ((double) size)));

				return resultNode;
			}
		});
		return result;
	}

	public static boolean isSameColumnCommentExist(String text)
	{
		boolean exist = false;
		if (StringUtils.isNotBlank(text))
		{
			try {
				List<Map<String, Object>> comments = getJdbcTemplate().queryForList(
					CHECK_IF_COLUMN_COMMENT_EXIST,
					text,
					text);
				if (comments != null && comments.size() > 0)
				{
					exist = true;
				}
			} catch(DataAccessException e) {
				Logger.error("Dataset isSameColumnCommentExist text is " + text);
				Logger.error("Exception = " + e.getMessage());
			}
		}
		return exist;
	}

	public static String postColumnComment(int datasetId, int columnId, Map<String, String[]> params, String user)
	{
    String result = "Post comment failed. Please try again.";
		if (params == null || params.size() == 0)
		{
			return result;
		}

		String text = "";
		if (params.containsKey("text")) {
			String[] textArray = params.get("text");
			if (textArray != null && textArray.length > 0)
			{
				text = textArray[0];
			}
		}
		if (StringUtils.isBlank(text))
		{
			return "Please input valid comment.";
		}

		Long commentId = 0L;
		if (params.containsKey("id")) {
			String[] idArray = params.get("id");
			if (idArray != null && idArray.length > 0)
			{
				String idStr = idArray[0];
				try
				{
					commentId = Long.parseLong(idStr);
				}
				catch(NumberFormatException e)
				{
					Logger.error("DatasetDAO postColumnComment wrong id parameter. Error message: " +
							e.getMessage());
					commentId = 0L;
				}
			}
		}

		Integer userId = 0;
		try
		{
			userId = (Integer)getJdbcTemplate().queryForObject(
					GET_USER_ID,
					Integer.class,
					user);
		}
		catch(EmptyResultDataAccessException e)
		{
			Logger.error("Dataset postColumnComment get user id failed, username = " + user);
			Logger.error("Exception = " + e.getMessage());
		}

		if (userId != null && userId !=0)
		{
			if (commentId != null && commentId != 0)
			{
				int row = getJdbcTemplate().update(UPDATE_COLUMN_COMMENT, text, commentId);
				if (row > 0)
				{
					result = "";
				}
			}
			else
			{
				if (isSameColumnCommentExist(text))
				{
					return "Same comment already exists.";
				}
				KeyHolder keyHolder = new GeneratedKeyHolder();
				final String comment = text;
				final int authorId = userId;
				getJdbcTemplate().update(
						new PreparedStatementCreator() {
							public PreparedStatement createPreparedStatement(Connection con) throws SQLException {
								PreparedStatement pst =
										con.prepareStatement(CREATE_COLUMN_COMMENT, new String[]{"id"});
								pst.setString(1, comment);
								pst.setInt(2, authorId);
								pst.setString(3, comment);
								return pst;
							}
						},
						keyHolder);
				commentId =  (Long)keyHolder.getKey();
				result = "";
			}
		}

		try
		{
			getJdbcTemplate().update(
				CREATE_DATASET_COLUMN_COMMENT_REFERENCE,
					datasetId,
					columnId,
					commentId
			);
		}
		catch(DataAccessException e)
		{
			Logger.error("Dataset postColumnComment insert ignore reference, datasetId = " +
					Integer.toString(datasetId) + " columnId = " + Integer.toString(columnId));
			Logger.error("Exception = " + e.getMessage());
		}

		List<Map<String, Object>> defaultComment = null;
		try {
			defaultComment = getJdbcTemplate().queryForList(
					CHECK_COLUMN_COMMENT_HAS_DEFAULT,
					datasetId,
					columnId
			) ;
		} catch(DataAccessException e) {
			Logger.error("Dataset postColumnComment - check for default, datasetId = " +
					Integer.toString(datasetId) + " columnId = " + Integer.toString(columnId));
			Logger.error("Exception = " + e.getMessage());
		}

		Boolean hasDefault = false;
		if(defaultComment.size() > 0) {
			hasDefault = true;
		}

		if(hasDefault) {
			result = "";
		} else {
			try {
				getJdbcTemplate().update(
						SET_COLUMN_COMMENT_DEFAULT,
						datasetId,
						columnId,
						commentId
				);
				result = "";
			} catch(DataAccessException e) {
				result = "Post comment failed. Please try again.";
				Logger.error("Dataset postColumnComment set default comment, datasetId = " +
						Integer.toString(datasetId) + " columnId = " + Integer.toString(columnId));
				Logger.error("Exception = " + e.getMessage());
			}
		}
		return result;
	}

	public static boolean deleteColumnComment(int datasetId, int columnId, int id)
	{
		boolean result = false;

		Integer commentCount = getJdbcTemplate().queryForObject(
			GET_COUNT_COLUMN_COMMENTS_BY_ID,
			new Object[] {datasetId, id},
			Integer.class);

		if(commentCount == null || commentCount == 0)
		{
			result = false;
		}
		else if (commentCount == 1)
		{
			try
			{
				getJdbcTemplate().update(
					DELETE_COLUMN_COMMENT_AND_REFERENCE,
					datasetId,
					columnId,
					id
				);
			}
			catch(DataAccessException e) {
				result = false;
				Logger.error("Dataset deleteColumnComment remove reference and comment, datasetId = " +
					Integer.toString(datasetId) + " columnId = " + Integer.toString(columnId));
				Logger.error("Exception = " + e.getMessage());
			}
		}
		else {
			try {
				getJdbcTemplate().update(
					DELETE_COLUMN_COMMENT_REFERENCE,
					datasetId,
					columnId,
					id
				);
			}
			catch(DataAccessException e) {
				result = false;
				Logger.error("Dataset deleteColumnComment remove reference, datasetId = " +
					Integer.toString(datasetId) + " columnId = " + Integer.toString(columnId));
				Logger.error("Exception = " + e.getMessage());
			}
		}

		return result;
	}

	public static List similarColumnComments(Long datasetId, int columnId)
	{
		List<SimilarComments> comments = new ArrayList<SimilarComments>();
		List<Map<String, Object>> rows = null;
		String fieldName = "";
		try {
			fieldName = (String)getJdbcTemplate().queryForObject(
					GET_COLUMN_NAME_BY_ID,
					String.class,
					columnId);
		} catch(DataAccessException e) {
			Logger.error("Dataset similarColumnComments - get field name for columnId, datasetId = " +
					Long.toString(datasetId) + " columnId = " + Integer.toString(columnId));
			Logger.error("Exception = " + e.getMessage());
			return comments;
		}

		try {
			rows = getJdbcTemplate().queryForList(
					GET_SIMILAR_COMMENTS_BY_FIELD_NAME,
					fieldName
			);
			for(Map row : rows) {

				SimilarComments sc = new SimilarComments();
				sc.count = (Long)row.get("count");
				sc.commentId = (Long)row.get("comment_id");
				sc.comment = (String)row.get("comment");
				sc.datasetId = datasetId;
				comments.add(sc);
			}
		} catch(DataAccessException e) {
			Logger.error("Dataset similarColumnComments - get comments by field name, datasetId = " +
					Long.toString(datasetId) + " columnId = " + Integer.toString(columnId));
			Logger.error("Exception = " + e.getMessage());
			return comments;
		}
		return comments;
	}

	public static boolean assignColumnComment(int datasetId, int columnId, int commentId)
	{
		Boolean result = false;
		try {
			getJdbcTemplate().update(
					SET_COLUMN_COMMENT_TO_FALSE,
					datasetId,
					columnId
			);
		} catch(DataAccessException e) {
			Logger.error("Dataset assignColumnComment - set current default to false, datasetId = " +
					Integer.toString(datasetId) + " columnId = " + Integer.toString(columnId));
			Logger.error("Exception = " + e.getMessage());
			return result;
		}

		try {
			getJdbcTemplate().update(
					INSERT_DATASET_COLUMN_COMMENT,
					datasetId,
					columnId,
					commentId
			);
			result = true;
		} catch(DataAccessException e) {
			Logger.error("Dataset assignColumnComment - set current default to false, datasetId = " +
					Integer.toString(datasetId) + " columnId = " + Integer.toString(columnId));
			Logger.error("Exception = " + e.getMessage());
			result = false;
		}
		return result;
	}

	public static List similarColumns(int datasetId, int columnId)
	{
		List<SimilarColumns> columns = new ArrayList<SimilarColumns>();
		List<Map<String, Object>> rows = null;
		String fieldName = "";
		try {
			fieldName = (String)getJdbcTemplate().queryForObject(
					GET_COLUMN_NAME_BY_ID,
					String.class,
					columnId);
		} catch(DataAccessException e) {
			Logger.error("Dataset similarColumns - get field name for columnId, datasetId = " +
					Integer.toString(datasetId) + " columnId = " + Integer.toString(columnId));
			Logger.error("Exception = " + e.getMessage());
			return columns;
		}
		try {
			rows = getJdbcTemplate().queryForList(
					GET_SIMILAR_COLUMNS_BY_FIELD_NAME,
					datasetId,
					fieldName
			);
			for(Map row : rows) {
				SimilarColumns sc = new SimilarColumns();
				sc.datasetId = (Long)row.get("dataset_id");
				sc.datasetName = (String)row.get("dataset_name");
				sc.columnId = (Long)row.get("field_id");
				sc.dataType = (String)row.get("data_type");
				sc.source = (String)row.get("source");
				sc.commentId = (Long)row.get("comment_id");
				sc.comment = (String)row.get("comment");
				columns.add(sc);
			}
		} catch(DataAccessException e) {
			Logger.error("Dataset similarColumns - get columns by field name, datasetId = " +
					Integer.toString(datasetId) + " columnId = " + Integer.toString(columnId));
			Logger.error("Exception = " + e.getMessage());
			return columns;
		}
		return columns;
	}

	private static void updateDatasetOwnerDatabase(int datasetId, String datasetUrn, List<DatasetOwner> owners)
	{
		getJdbcTemplate().batchUpdate(UPDATE_DATASET_CONFIRMED_OWNERS,
				new BatchPreparedStatementSetter() {
					@Override
					public void setValues(PreparedStatement ps, int i)
							throws SQLException {
						DatasetOwner owner = owners.get(i);
						ps.setInt(1, datasetId);
						ps.setString(2, owner.userName);
						ps.setInt(3, owner.isGroup ? 301 : 300);
						ps.setString(4, owner.namespace);
						ps.setString(5, owner.type);
						ps.setString(6, owner.isGroup ? "Y" : "N");
						ps.setString(7, owner.isActive != null && owner.isActive ? "Y" : "N");
						ps.setInt(8, owner.sortId);
						ps.setString(9, datasetUrn);
						ps.setString(10, owner.subType);
						ps.setString(11, owner.idType);
						ps.setString(12, owner.source);
						ps.setString(13, owner.confirmedBy);
						if (StringUtils.isBlank(owner.confirmedBy)) {
							ps.setLong(14, 0L);
						} else {
							ps.setLong(14, Instant.now().getEpochSecond());
						}
						ps.setString(15, owner.type);
						ps.setString(16, owner.isGroup ? "Y" : "N");
						ps.setInt(17, owner.sortId);
						ps.setString(18, owner.subType);
						ps.setString(19, owner.confirmedBy);
						if (StringUtils.isBlank(owner.confirmedBy)) {
							ps.setLong(20, 0L);
						} else {
							ps.setLong(20, Instant.now().getEpochSecond());
						}
					}

					@Override
					public int getBatchSize()
					{
						return owners.size();
					}
				}
		);
	}

	public static ReturnCode updateDatasetOwners(int datasetId, Map<String, String[]> ownersMap, String user) {
		// ownerMap should contain mapping 'owners': ['ownerInfoJsonString']
		if (ownersMap == null || !ownersMap.containsKey("owners") || ownersMap.get("owners") == null
				|| ownersMap.get("owners").length == 0) {
			return ReturnCode.MissingFields;
		}

		String[] textArray = ownersMap.get("owners");
		final JsonNode node = Json.parse(textArray[0]);

		List<DatasetOwner> owners = new ArrayList<>();
		int confirmedOwnerUserCount = 0;

		for(int i = 0; i < node.size(); i++) {
			final JsonNode ownerNode = node.get(i);
			if (ownerNode != null) {
				String userName = ownerNode.has("userName") ? ownerNode.get("userName").asText() : "";
				if (StringUtils.isBlank(userName)) {
					continue;
				}

				Boolean isGroup = ownerNode.has("isGroup") && ownerNode.get("isGroup").asBoolean();
				String type = ownerNode.has("type") && !ownerNode.get("type").isNull() ? ownerNode.get("type").asText() : "";
				String subType =
						ownerNode.has("subType") && !ownerNode.get("subType").isNull() ? ownerNode.get("subType").asText() : "";
				String confirmedBy =
						ownerNode.has("confirmedBy") && !ownerNode.get("confirmedBy").isNull()
								? ownerNode.get("confirmedBy").asText() : "";
				String idType = ownerNode.has("idType") ? ownerNode.get("idType").asText() : "";
				String source = ownerNode.has("source") ? ownerNode.get("source").asText() : "";

				DatasetOwner owner = new DatasetOwner();
				owner.userName = userName;
				owner.isGroup = isGroup;
				owner.namespace = isGroup ? "urn:li:griduser" : "urn:li:corpuser";
				owner.type = type;
				owner.subType = subType;
				owner.idType = idType;
				owner.source = source;
				owner.confirmedBy = confirmedBy;
				owner.sortId = i;
				owners.add(owner);

				if (type.equalsIgnoreCase("owner") && idType.equalsIgnoreCase("user") && StringUtils.isNotBlank(confirmedBy)) {
					confirmedOwnerUserCount++;
				}
			}
		}

    // enforce at least two confirmed owner for a dataset before making any changes
		if (confirmedOwnerUserCount < 2) {
			return ReturnCode.RuleViolation;
		}

		// first mark existing owners as deleted, new owners will be updated later
		getJdbcTemplate().update(MARK_DATASET_OWNERS_AS_DELETED, datasetId);

		if (owners.size() > 0) {
			String urn = getDatasetUrnById(datasetId);
			updateDatasetOwnerDatabase(datasetId, urn, owners);
		}
		return ReturnCode.Success;
	}

	public static void getDependencies(
			Long datasetId,
			List<DatasetDependency> depends)
	{
		String nativeName = null;
		try
		{
			nativeName = getJdbcTemplate().queryForObject(
					GET_DATASET_NATIVE_NAME,
					String.class,
					datasetId);
		}
		catch (EmptyResultDataAccessException e)
		{
			nativeName = null;
		}

		if (StringUtils.isNotBlank(nativeName))
		{
			getDatasetDependencies("/" + nativeName.replace(".", "/"), 1, 0, depends);
		}

	}

	public static void getReferences(
			Long datasetId,
			List<DatasetDependency> references)
	{
		String nativeName = null;
		try
		{
			nativeName = getJdbcTemplate().queryForObject(
					GET_DATASET_NATIVE_NAME,
					String.class,
					datasetId);
		}
		catch (EmptyResultDataAccessException e)
		{
			nativeName = null;
		}

		if (StringUtils.isNotBlank(nativeName))
		{
			getDatasetReferences("/" + nativeName.replace(".", "/"), 1, 0, references);
		}

	}


	public static void getDatasetDependencies(
			String objectName,
			int level,
			int parent,
			List<DatasetDependency> depends)
	{
		if (depends == null)
		{
			depends = new ArrayList<DatasetDependency>();
		}

		List<Map<String, Object>> rows = null;
		rows = getJdbcTemplate().queryForList(
				GET_DATASET_DEPENDS_VIEW,
				objectName);

		if (rows != null)
		{
			for (Map row : rows) {
				DatasetDependency dd = new DatasetDependency();
				dd.datasetId = (Long) row.get("mapped_object_dataset_id");
				dd.objectName = (String) row.get("mapped_object_name");
				dd.objectType = (String) row.get("mapped_object_type");
				dd.objectSubType = (String) row.get("mapped_object_sub_type");
				if (dd.datasetId != null && dd.datasetId > 0)
				{
					dd.isValidDataset = true;
					dd.datasetLink = "#/datasets/" + Long.toString(dd.datasetId);
				}
				else
				{
					dd.isValidDataset = false;
				}
				dd.level = level;
				dd.sortId = depends.size() + 1;
				dd.treeGridClass = "treegrid-" + Integer.toString(dd.sortId);
				if (parent != 0)
				{
					dd.treeGridClass += " treegrid-parent-" + Integer.toString(parent);
				}
				depends.add(dd);
				getDatasetDependencies(dd.objectName, level + 1, dd.sortId, depends);
			}
		}
	}

	public static void getDatasetReferences(
			String objectName,
			int level,
			int parent,
			List<DatasetDependency> references)
	{
		if (references == null)
		{
			references = new ArrayList<DatasetDependency>();
		}

		List<Map<String, Object>> rows = null;
		rows = getJdbcTemplate().queryForList(
				GET_DATASET_REFERENCES,
				objectName);

		if (rows != null)
		{
			for (Map row : rows) {
				DatasetDependency dd = new DatasetDependency();
				dd.datasetId = (Long) row.get("object_dataset_id");
				dd.objectName = (String) row.get("object_name");
				dd.objectType = (String) row.get("object_type");
				dd.objectSubType = (String) row.get("object_sub_type");
				if (dd.datasetId != null && dd.datasetId > 0)
				{
					dd.isValidDataset = true;
					dd.datasetLink = "#/datasets/" + Long.toString(dd.datasetId);
				}
				else
				{
					dd.isValidDataset = false;
				}
				dd.level = level;
				dd.sortId = references.size() + 1;
				dd.treeGridClass = "treegrid-" + Integer.toString(dd.sortId);
				if (parent != 0)
				{
					dd.treeGridClass += " treegrid-parent-" + Integer.toString(parent);
				}
				references.add(dd);
				getDatasetReferences(dd.objectName, level + 1, dd.sortId, references);
			}
		}
	}

	public static List<DatasetListViewNode> getDatasetListViewNodes(String urn) {

		List<DatasetListViewNode> nodes = new ArrayList<DatasetListViewNode>();
		List<Map<String, Object>> rows = null;

		if (StringUtils.isBlank(urn)) {
			rows = getJdbcTemplate().queryForList(
					GET_DATASET_LISTVIEW_TOP_LEVEL_NODES);
		} else {
			rows = getJdbcTemplate().queryForList(
					GET_DATASET_LISTVIEW_NODES_BY_URN,
					urn,
					urn,
					urn,
					urn,
					urn,
					urn + "%");
		}

		for (Map row : rows) {

			DatasetListViewNode node = new DatasetListViewNode();
			node.datasetId = (Long) row.get(DatasetWithUserRowMapper.DATASET_ID_COLUMN);
			node.nodeName = (String) row.get(DatasetWithUserRowMapper.DATASET_NAME_COLUMN);
			String nodeUrn = (String) row.get(DatasetWithUserRowMapper.DATASET_URN_COLUMN);
			if (node.datasetId != null && node.datasetId > 0)
			{
				node.nodeUrl = "#/datasets/" + node.datasetId;
			}
			else
			{
				node.nodeUrl = "#/datasets/name/" + node.nodeName + "/page/1?urn=" + nodeUrn;
			}
			nodes.add(node);
		}

		return nodes;
	}

	public static List<String> getDatasetVersions(Long datasetId, Integer dbId)
	{
		return getJdbcTemplate().queryForList(GET_DATASET_VERSIONS, String.class, datasetId);
	}

	public static String getDatasetSchemaTextByVersion(
			Long datasetId, String version)
	{
		String schemaText = null;
		try
		{
			schemaText = getJdbcTemplate().queryForObject(
					GET_DATASET_SCHEMA_TEXT_BY_VERSION,
					String.class,
					datasetId, version);
		}
		catch (EmptyResultDataAccessException e)
		{
			schemaText = null;
		}
		return schemaText;
	}

	public static List<DatasetInstance> getDatasetInstances(Long id)
	{
		List<DatasetInstance> datasetInstances = new ArrayList<DatasetInstance>();

		List<Map<String, Object>> rows = null;
		rows = getJdbcTemplate().queryForList(
				GET_DATASET_INSTANCES,
				id);

		if (rows != null)
		{
			for (Map row : rows) {
				DatasetInstance datasetInstance = new DatasetInstance();
				datasetInstance.datasetId = id;
				datasetInstance.dbId = (Integer) row.get("db_id");
				datasetInstance.dbCode = (String) row.get("db_code");
				datasetInstances.add(datasetInstance);
			}
		}
		return datasetInstances;
	}

	public static List<String> getDatasetPartitionGains(Long id) {
		return getJdbcTemplate().queryForList(GET_DATASET_ACCESS_PARTITION_GAIN, String.class, id);
	}

	public static List<String> getDatasetPartitionInstance(Long id, String partition) {
		return getJdbcTemplate().queryForList(GET_DATASET_ACCESS_PARTITION_INSTANCES, String.class, id, partition);
	}

	public static List<DatasetPartition> getDatasetAccessibilty(Long id) {

		ObjectNode resultNode = Json.newObject();
		List<String> partitions = getDatasetPartitionGains(id);
		List<DatasetPartition> datasetPartitions = new ArrayList<DatasetPartition>();
		if (partitions != null && partitions.size() > 0)
		{
			for(String partition:partitions)
			{
				List<Map<String, Object>> rows = null;
				Map<String, DatasetAccessibility> addedAccessibilities= new HashMap<String, DatasetAccessibility>();
				rows = getJdbcTemplate().queryForList(
						GET_DATASET_ACCESS,
						id,
						partition);
				List<DatasetAccessibility> datasetAccessibilities = new ArrayList<DatasetAccessibility>();
				List<String> instances = new ArrayList<String>();
				instances = getDatasetPartitionInstance(id, partition);

				if (rows != null)
				{
					for (Map row : rows) {
						DatasetAccessibility datasetAccessibility = new DatasetAccessibility();
						datasetAccessibility.datasetId = id;
						datasetAccessibility.itemList = new ArrayList<DatasetAccessItem>();
						datasetAccessibility.dbId = (Integer) row.get("db_id");
						datasetAccessibility.dbName = (String) row.get("db_code");
						datasetAccessibility.datasetType = (String) row.get("dataset_type");
						datasetAccessibility.partitionExpr = (String) row.get("partition_expr");
						datasetAccessibility.partitionGain = partition;
						datasetAccessibility.dataTimeExpr = (String) row.get("data_time_expr");
						datasetAccessibility.dataTimeEpoch = (Integer) row.get("data_time_epoch");
						datasetAccessibility.recordCount = (Long) row.get("record_count");
						if (datasetAccessibility.recordCount == null)
						{
							datasetAccessibility.recordCount = 0L;
						}
						datasetAccessibility.sizeInByte = (Long) row.get("size_in_byte");
						datasetAccessibility.logTimeEpoch = (Integer) row.get("log_time_epoch");
						datasetAccessibility.logTimeEpochStr = row.get("log_time_str").toString();
						DatasetAccessibility exist = addedAccessibilities.get(datasetAccessibility.dataTimeExpr);
						if (exist == null)
						{
							for(int i = 0; i < instances.size(); i++)
							{
								DatasetAccessItem datasetAccessItem = new DatasetAccessItem();
								if(instances.get(i).equalsIgnoreCase(datasetAccessibility.dbName))
								{
									datasetAccessItem.recordCountStr = Long.toString(datasetAccessibility.recordCount);
									datasetAccessItem.logTimeEpochStr = datasetAccessibility.logTimeEpochStr;
									datasetAccessItem.isPlaceHolder = false;
								}
								else
								{
									datasetAccessItem.recordCountStr = "";
									datasetAccessItem.logTimeEpochStr = "";
									datasetAccessItem.isPlaceHolder = true;
								}
								datasetAccessibility.itemList.add(datasetAccessItem);
							}
							addedAccessibilities.put(datasetAccessibility.dataTimeExpr, datasetAccessibility);
							datasetAccessibilities.add(datasetAccessibility);
						}
						else
						{
							for(int i = 0; i < instances.size(); i++)
							{
								if(instances.get(i).equalsIgnoreCase(datasetAccessibility.dbName))
								{
									DatasetAccessItem datasetAccessItem = new DatasetAccessItem();
									datasetAccessItem.logTimeEpochStr = datasetAccessibility.logTimeEpochStr;
									datasetAccessItem.recordCountStr = Long.toString(datasetAccessibility.recordCount);
									datasetAccessItem.isPlaceHolder = false;
									exist.itemList.set(i, datasetAccessItem);
								}
							}
						}
					}
				}
				DatasetPartition datasetPartition = new DatasetPartition();
				datasetPartition.datasetId = id;
				datasetPartition.accessibilityList = datasetAccessibilities;
				datasetPartition.instanceList = instances;
				datasetPartition.partition = partition;
				datasetPartitions.add(datasetPartition);
			}
		}

		return datasetPartitions;
	}

	public static DatasetCompliance getDatasetComplianceByDatasetId(int datasetId) throws Exception {
		Map<String, Object> result = getJdbcTemplate().queryForMap(GET_DATASET_COMPLIANCE_BY_DATASET_ID, datasetId);

		return maptoDatasetCompliance(result);
	}

	public static DatasetCompliance getDatasetComplianceByDatasetUrn(String datasetUrn) throws Exception {
		Map<String, Object> result = getJdbcTemplate().queryForMap(GET_DATASET_COMPLIANCE_BY_URN, datasetUrn);

		return maptoDatasetCompliance(result);
	}

	private static DatasetCompliance maptoDatasetCompliance(Map<String, Object> result) throws IOException {
		ObjectMapper om = new ObjectMapper();
		List<DatasetFieldEntity> entities =
				om.readValue((String) result.get("compliance_purge_entities"), new TypeReference<List<DatasetFieldEntity>>() {
				});

		DatasetCompliance record = new DatasetCompliance();
		record.setDatasetId(((Long) result.get("dataset_id")).intValue());
		record.setDatasetUrn((String) result.get("dataset_urn"));
		record.setComplianceType((String) result.get("compliance_purge_type"));
		record.setCompliancePurgeEntities(entities);
		record.setModifiedTime((long) result.get("modified_time"));
		return record;
	}

	public static void updateDatasetCompliancePolicy(int datasetId, JsonNode node) throws Exception {
		ObjectMapper om = new ObjectMapper();
		DatasetCompliance record = om.convertValue(node, DatasetCompliance.class);
		if (record.getDatasetId() != null && datasetId != record.getDatasetId()) {
			throw new IllegalArgumentException("Dataset id doesn't match.");
		}

		String urn = record.getDatasetUrn() != null ? record.getDatasetUrn() : getDatasetUrnById(datasetId);

		Map<String, Object> parameters = new HashMap<>();
		parameters.put("id", datasetId);
		parameters.put("urn", urn);
		parameters.put("type", record.getComplianceType());
		parameters.put("entities", om.writeValueAsString(record.getCompliancePurgeEntities()));
		parameters.put("modified", System.currentTimeMillis() / 1000);
		getNamedParameterJdbcTemplate().update(INSERT_DATASET_COMPLIANCE, parameters);
	}

	public static DatasetSecurity getDatasetSecurityByDatasetId(int datasetId) throws Exception {
		Map<String, Object> result = getJdbcTemplate().queryForMap(GET_DATASET_SECURITY_BY_DATASET_ID, datasetId);

		return mapToDatasetSecurity(result);
	}

	public static DatasetSecurity getDatasetSecurityByDatasetUrn(String datasetUrn) throws Exception {
		Map<String, Object> result = getJdbcTemplate().queryForMap(GET_DATASET_SECURITY_BY_URN, datasetUrn);

		return mapToDatasetSecurity(result);
	}

	private static DatasetSecurity mapToDatasetSecurity(Map<String, Object> result) throws IOException {
		DatasetSecurity record = new DatasetSecurity();
		record.setDatasetId(((Long) result.get("dataset_id")).intValue());
		record.setDatasetUrn((String) result.get("dataset_urn"));
		record.setDatasetClassification(
				jsonToObject((String) result.get("dataset_classification"), new TypeReference<Map<String, Object>>() {
				}));
		record.setConfidentiality((String) result.get("confidentiality"));
		record.setClassification(
				jsonToObject((String) result.get("classification"), new TypeReference<Map<String, List<DatasetFieldEntity>>>() {
				}));
		record.setRecordOwnerType((String) result.get("record_owner_type"));
		record.setRetentionPolicy(
				jsonToObject((String) result.get("retention_policy"), new TypeReference<Map<String, Object>>() {
				}));
		record.setGeographicAffinity(
				jsonToObject((String) result.get("geographic_affinity"), new TypeReference<Map<String, Object>>() {
				}));
		record.setModifiedTime((long) result.get("modified_time"));
		return record;
	}

	private static <T> T jsonToObject(String jsonString, TypeReference type) throws IOException {
		ObjectMapper om = new ObjectMapper();
		return StringUtils.isBlank(jsonString) ? null : om.readValue(jsonString, type);
	}

	public static void updateDatasetSecurityInfo(int datasetId, JsonNode node) throws Exception {

		ObjectMapper om = new ObjectMapper();
		DatasetSecurity record = om.convertValue(node, DatasetSecurity.class);
		if (record.getDatasetId() != null && datasetId != record.getDatasetId()) {
			throw new IllegalArgumentException("Dataset id doesn't match.");
		}

		String urn = record.getDatasetUrn() != null ? record.getDatasetUrn() : getDatasetUrnById(datasetId);

		Map<String, Object> parameters = new HashMap<>();
		parameters.put("id", datasetId);
		parameters.put("urn", urn);
		parameters.put("dataset_classification", om.writeValueAsString(record.getDatasetClassification()));
		parameters.put("confidentiality", record.getConfidentiality());
		parameters.put("classification", om.writeValueAsString(record.getClassification()));
		parameters.put("ownerType", record.getRecordOwnerType());
		parameters.put("policy", om.writeValueAsString(record.getRetentionPolicy()));
		parameters.put("geo", om.writeValueAsString(record.getGeographicAffinity()));
		parameters.put("modified", System.currentTimeMillis() / 1000);
		getNamedParameterJdbcTemplate().update(INSERT_DATASET_SECURITY, parameters);
	}
}