Grammar Composition System

/src/agree-sys/90-itsdb.cs

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Collections;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Text;

using miew.ReadOnly;
using miew.Binding;
using miew.Reflection;
using agree;
using System.Collections.ObjectModel;

namespace agree.itsdb
{
	using Type = System.Type;

	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// 
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	public interface IitsdbTable : IBindingList, ISysObj
	{
		Type ItsdbType { get; }

		void Load(String s_dir, IList<String> schema);

		void ValidateExternalKeys();
	};

	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// 
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	public class ItsdbDatabase : List<IitsdbTable>, ISysObj
	{
		readonly static public Type[] ttypes = 
		{
			typeof(ItsdbItemTable),
			typeof(ItsdbAnalysisTable),
			typeof(ItsdbPhenomenonTable),
			typeof(ItsdbParameterTable),
			typeof(ItsdbSetTable),
			typeof(ItsdbItem_PhenomenonTable),
			typeof(ItsdbItem_SetTable),
			typeof(ItsdbRunTable),
			typeof(ItsdbParseTable),
			typeof(ItsdbResultTable),
			typeof(ItsdbRuleTable),
			typeof(ItsdbOutputTable),
			typeof(ItsdbEdgeTable),
			typeof(ItsdbTreeTable),
			typeof(ItsdbDecisionTable),
			typeof(ItsdbPreferenceTable),
			typeof(ItsdbUpdateTable),
			typeof(ItsdbFoldTable),
			typeof(ItsdbScoreTable),
		};

		public Dictionary<String, IitsdbTable> table_name_lookup = new Dictionary<String, IitsdbTable>(StringComparer.OrdinalIgnoreCase);

		public IList<IitsdbTable> Tables { get { return this; } }

		public ItsdbItemTable ItemTable { get { return (ItsdbItemTable)table_name_lookup["item"]; } }
		public ItsdbAnalysisTable AnalysisTable { get { return (ItsdbAnalysisTable)table_name_lookup["analysis"]; } }
		public ItsdbParseTable ParseTable { get { return (ItsdbParseTable)table_name_lookup["parse"]; } }
		public ItsdbResultTable ResultsTable { get { return (ItsdbResultTable)table_name_lookup["result"]; } }
		public ItsdbRunTable RunTable { get { return (ItsdbRunTable)table_name_lookup["run"]; } }

		public String SourceDirectory { get { return s_dir; } }

		String s_dir;
		ISysObj so;

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// 
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		public IitsdbTable TableFromAnonymousItems(String name, String description, IEnumerable data)
		{
			Object prototype = data.OfType<Object>().First();

			var pi = AnonymousTypePromoter.GetPromotionInfo(
						name, 
						prototype,
						new String[] { "System.ComponentModel", "agree.itsdb" },
						new String[] { "System.dll", Assembly.GetAssembly(typeof(ItsdbItemType)).Location },
						new Type[] { typeof(ItsdbItemType) });

			Type T_tab = typeof(ItsdbTable<>).MakeGenericType(new Type[] { pi.Type });

			IitsdbTable t = (IitsdbTable)Activator.CreateInstance(T_tab, this, name, description);

			foreach (Object o in data)
				t.Add(pi.Promote(o));

			this.Add(t);
			return t;
		}


		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// Construct an Itsdb database over the specified directory
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		public ItsdbDatabase(ISysObj so, String s_dir)
			:base(ttypes.Length)
		{
			this.so = so;
			this.s_dir = s_dir;

			Object[] ctor_args = new Object[] { this };
			for (int i = 0; i < ttypes.Length; i++)
			{
				IitsdbTable t = (IitsdbTable)Activator.CreateInstance(ttypes[i], ctor_args);
				this.Add(t);
				table_name_lookup.Add(t.SysObjName, t);
			}

			String relations_file = Path.Combine(s_dir, "relations");
			if (!File.Exists(relations_file))
			{
				String msg = String.Format("The filename 'relations' could not be found in the directory '{0}'", s_dir);
				throw new FileNotFoundException(msg, "relations");
			}

			List<String> schema = new List<String>();
			IitsdbTable cur = null;

			foreach (String _l in File.ReadAllLines(relations_file))
			{
				String l = _l;
				int ix = l.IndexOf('#');
				if (ix != -1)
					l = l.Remove(ix);
				l = l.Trim();
				if (l == String.Empty)
					continue;

				String[] rgs = l.Split(default(Char[]), StringSplitOptions.RemoveEmptyEntries);
				if (rgs.Length == 1)
				{
					if (cur != null)
					{
						cur.Load(s_dir, schema);
						schema.Clear();
					}

					String s_tab = rgs[0].Trim(':');
					if (!table_name_lookup.TryGetValue(s_tab, out cur))
					{
						String msg = String.Format("The relations file refers to an unknown table type '{0}'", s_tab);
						throw new Exception(msg);
					}
				}
				else
					schema.Add(rgs[0]);
			}

			/// do the last table
			if (cur != null)
				cur.Load(s_dir, schema);

			/// now that all tables are loaded, validate external keys
			//foreach (IitsdbTable t in tables)
			//    t.ValidateExternalKeys();
		}

		public IitsdbTable GetTableFromItemType(Type t)
		{
			return this.First(tt => tt.ItsdbType == t);
		}

		public string SysObjName
		{
			get { return s_dir; }
		}

		public string SysObjDescription
		{
			get { return String.Format("[incr tsdb()] database : {0}", s_dir); }
		}

		public IReadOnlyDictionary<String, ISysObj> SysObjChildren
		{
			get { return new CovariantDictionaryWrapper<String, ISysObj, IitsdbTable>(table_name_lookup); }
		}

		public ISysObj SysObjParent
		{
			get { return so; }
		}
	};

	/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	///// <summary>
	///// 
	///// </summary>
	/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	//public class ItsdbJoinedTable<T> : ListItemBindHelper<T>, IitsdbJoinedTable
	//{
	//    ItsdbDatabase db;
	//    String name;
	//    String description;
	//    Type T_item;

	//    public ItsdbJoinedTable(ItsdbDatabase db, String name, String description, IEnumerable<T> data)
	//        :base(data)
	//    {
	//        //data.OfType<Object>().ToList()
	//        this.db = db;
	//        this.name = name;
	//        this.description = description;
	//        //this.T_item = this.Count > 0 ? this[0].GetType() : typeof(Object);
	//    }

	//    public string SysObjName
	//    {
	//        get { return name; }
	//    }

	//    public string SysObjDescription
	//    {
	//        get { return description; }
	//    }

	//    public IReadOnlyDictionary<String, ISysObj> SysObjChildren
	//    {
	//        get { return SysObjHelper<ISysObj>.Empty; }
	//    }

	//    public ISysObj SysObjParent
	//    {
	//        get { return db; }
	//    }

	//    public Type ItsdbType
	//    {
	//        get { return T_item; }
	//    }
	//};

	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// Template class for table of Itsb items of some strong-type
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	public class ItsdbTable<T> : BindingList<T>, IitsdbTable where T : ItsdbItemType
	{
		public ItsdbDatabase db;
		String name;
		String description;

		public ItsdbTable(ItsdbDatabase db)
		{
			this.db = db;
		}

		public ItsdbTable(ItsdbDatabase db, String name, String description)
			:this(db)
		{
			this.name = name;
			this.description = description;
		}

		public Type ItsdbType { get { return typeof(T); } }

		/// <summary>
		/// Using the ordered list of fields specified in 'schema,' load the table with data from a plaintext or gzip 
		/// file with the table's matching name, if any is found in the specified directory.
		/// </summary>
		public void Load(String s_dir, IList<String> schema)
		{
			var fields = schema.Select(s =>
			{
				FieldInfo fi = typeof(T).GetField("_" + s.Replace('-', '_'), BindingFlags.Instance | BindingFlags.NonPublic);
				if (fi == null)
				{
					String msg = String.Format("The field '{0}' in the schema for '{1}' is not recognized.", s, SysObjName);
					throw new Exception(msg);
				}
				return fi;
			}).ToArray();

			/// See if there's a data file for this table
			String db_file = Path.Combine(s_dir, SysObjName);
			Stream str;
			if (!File.Exists(db_file))
			{
				db_file += ".gz";
				if (!File.Exists(db_file))
					return;
			}

			/// If the data file is compressed, switch the stream to a gzip decoder
			str = File.Open(db_file, FileMode.Open, FileAccess.Read, FileShare.Read);
			if (db_file.EndsWith(".gz"))
				str = new GZipStream(str, CompressionMode.Decompress);

			/// Read lines from the data file into the table
			bool f_gave_warning = false;
			using (StreamReader sr = new StreamReader(str, Encoding.UTF8))
			{
				int i_l = 0;
				String line;
				while ((line = sr.ReadLine()) != null)
				{
					i_l++;
					/// Split the line of raw data into parts
					String[] data = line.Replace(@"\\", @"\").Split('@');
					if (data.Length > fields.Length)
					{
						String msg = String.Format("Number of data items does not match schema in file '{0}', line {1}", db_file, i_l);
						throw new Exception(msg);
					}
					else if (!f_gave_warning && data.Length < fields.Length)
					{
						//Console.WriteLine("warning: Number of data fields ({0}) is less than the number of fields in the schema ({1}) in file '{2}'", data.Length, fields.Length, db_file);
						f_gave_warning = true;
					}

					/// Create an item of the appropriate type and load it with the data from each field
					T o_item = Activator.CreateInstance<T>();
					for (int i = 0; i < data.Length; i++)
					{
						FieldInfo fi = fields[i];
						String d = data[i];

						/// Convert the string data into a strongly typed object of the appropriate type
						Object o_data;
						if (fi.FieldType == typeof(long))
						{
							long il;
							if (d == String.Empty)
								il = 0;
							else if (!long.TryParse(d, out il))
							{
								String msg = String.Format("Invalid long integer value '{0}' in file '{1}', line {2}", d, db_file, i_l);
								throw new Exception(msg);
							}
							o_data = il;
						}
						else if (fi.FieldType == typeof(int))
						{
							int ii;
							if (d == String.Empty)
								ii = 0;
							else if (!int.TryParse(d, out ii))
							{
								uint ui;
								if (!uint.TryParse(d, out ui))
								{
									String msg = String.Format("Invalid integer value '{0}' in file '{1}', line {2}", d, db_file, i_l);
									throw new Exception(msg);
								}
								ii = (int)ui;
							}
							o_data = ii;
						}
						else if (fi.FieldType == typeof(string))
							o_data = d;
						else if (fi.FieldType == typeof(DateTime))
						{
							String sd = new String(d.Where(ch => ch != '(' && ch != ')').ToArray());
							DateTime dt;
							if (sd == String.Empty)
								dt = default(DateTime);
							else if (!DateTime.TryParse(sd, CultureInfo.GetCultureInfo("de-DE"), DateTimeStyles.None, out dt))
							{
								String msg = String.Format("Unrecognized date/time format '{0}' in file '{1}', line {2}", d, db_file, i_l);
								throw new Exception(msg);
							}
							o_data = dt;
						}
						else
							throw new Exception();

						/// Set the field's value into the item
						fi.SetValue(o_item, o_data);
					}
					/// Add the item to the table
					this.Add(o_item);
				}
			}
			str.Dispose();

			/// setup primary keys, if any
			foreach (FieldInfo pkf in typeof(T).GetFields(BindingFlags.Instance | BindingFlags.NonPublic)
											.Where(fi => fi.GetCustomAttributes(true).Any(o => o is PrimaryKeyAttribute)))
			{
				if (pkf.FieldType != typeof(int))
					throw new Exception("primary key must be an integer");

				FieldInfo map_field = GetType().GetField(pkf.Name + "_map");
				Dictionary<int, T> map = (Dictionary<int, T>)map_field.GetValue(this);

#if DUPLICATE_KEY_INFO
				Dictionary<int, int> repeated_keys = new Dictionary<int, int>();
#endif
				foreach (T t in this)
				{
					int id = (int)pkf.GetValue(t);
					if (map.ContainsKey(id))
					{
#if DUPLICATE_KEY_INFO
						if (repeated_keys.ContainsKey(id))
						    repeated_keys[id]++;
						else
						    repeated_keys.Add(id, 2);
#endif
					}
					else
						map.Add(id, t);
				}
#if DUPLICATE_KEY_INFO
				if (repeated_keys.Count > 0)
				{
				    Console.WriteLine("warning: field '{0}' in table '{1}' has duplicate primary key value(s):", pkf.Name, db_file);
				    foreach (var kvp in repeated_keys.OrderBy(k => k.Key))
				        Console.WriteLine("\tvalue: {0} ({1} instances)", kvp.Key, kvp.Value);
				}
#endif
			}
		}

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// validate external keys via reflection
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		public void ValidateExternalKeys()
		{
			/// Process all fields in this table's item type which are marked with the 'ExternalKey' attribute
			foreach (var fa in typeof(T)
								.GetFields(BindingFlags.Instance | BindingFlags.NonPublic)
								.SelectMany(fi_this => fi_this.GetCustomAttributes(true)
															.OfType<ExternalKeyAttribute>()
															.Select(attr => new { fi_this, attr })))
			{
				/// Common name of the shared key
				String s_key = fa.fi_this.Name;

				/// type of external item which introduces the primary key
				Type ext_item_type = fa.attr.ti_ext;

				/// find a matching field name in the external item
				FieldInfo fi_ext = ext_item_type.GetField(s_key, BindingFlags.Instance | BindingFlags.NonPublic);
				if (fi_ext == null || fi_ext.GetCustomAttributes(typeof(PrimaryKeyAttribute), true).Length == 0)
				{
					String msg = String.Format("Couldn't resolve primary key '{0}' in table '{1}' referenced by external key in table '{2}'",
						s_key, ext_item_type, SysObjName);
					throw new Exception(msg);
				}

				/// get external table
				IitsdbTable ext_table = db.GetTableFromItemType(ext_item_type) as IitsdbTable;
				if (ext_table == null)
					continue;

				/// get external table type
				Type ext_table_type = ext_table.GetType();

				/// Get the map out of the external table
				Object map = ext_table_type.GetField(s_key + "_map", BindingFlags.Instance | BindingFlags.NonPublic)
												.GetValue(ext_table);

				/// find the 'ContainsKey' method for a generic dictionary containing the external item as a value
				MethodInfo mi_ContainsKey = typeof(Dictionary<,>)
												.MakeGenericType(new Type[] { typeof(int), ext_item_type })
												.GetMethod("ContainsKey");

				/// Check each item in this table
				HashSet<int> bad_ids = new HashSet<int>();
				foreach (T t in this)
				{
					/// get the secondary key value specified by an item
					int id = (int)fa.fi_this.GetValue(t);

					/// See if the primary contains the value
					if (!(bool)mi_ContainsKey.Invoke(map, new Object[] { id }))
						bad_ids.Add(id);
				}

				if (bad_ids.Count > 0)
				{
					Console.WriteLine();
					String msg = String.Format("The following value(s) specified for field '{0}' in table '{1}' do not match any value of that field in primary key table '{2}':",
						s_key.Substring(1).Replace('_', '-'),
						SysObjName,
						ext_table.SysObjName);
					Console.WriteLine(msg);
					Console.WriteLine(String.Join(", ", bad_ids.OrderBy(_i => _i)));
				}
			}
		}

		public String SysObjName
		{
			get { return typeof(T).Name.Replace("Itsdb", String.Empty).Replace('_', '-').ToLower(); }
		}

		public string SysObjDescription
		{
			get { return String.Format("{0} - {1}", db.SysObjName, SysObjName); }
		}

		public IReadOnlyDictionary<string, ISysObj> SysObjChildren
		{
			get { return SysObjHelper<ISysObj>.Empty; }
		}

		public ISysObj SysObjParent
		{
			get { return db; }
		}
	};

	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	///
	/// Itsdb table types follow 
	///
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

	public class ItsdbItemTable : ItsdbTable<ItsdbItem>
	{
		public ItsdbItemTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbItem> _i_id_map = new Dictionary<int, ItsdbItem>();
	};

	public class ItsdbAnalysisTable : ItsdbTable<ItsdbAnalysis>
	{
		public ItsdbAnalysisTable(ItsdbDatabase db) : base(db) { }

	};

	public class ItsdbPhenomenonTable : ItsdbTable<ItsdbPhenomenon>
	{
		public ItsdbPhenomenonTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbPhenomenon> _p_id_map = new Dictionary<int, ItsdbPhenomenon>();
	};

	public class ItsdbParameterTable : ItsdbTable<ItsdbParameter>
	{
		public ItsdbParameterTable(ItsdbDatabase db) : base(db) { }

	};

	public class ItsdbSetTable : ItsdbTable<ItsdbSet>
	{
		public ItsdbSetTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbSet> _s_id_map = new Dictionary<int, ItsdbSet>();
	};

	public class ItsdbItem_PhenomenonTable : ItsdbTable<ItsdbItem_Phenomenon>
	{
		public ItsdbItem_PhenomenonTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbItem_Phenomenon> _ip_id_map = new Dictionary<int, ItsdbItem_Phenomenon>();
	};

	public class ItsdbItem_SetTable : ItsdbTable<ItsdbItem_Set>
	{
		public ItsdbItem_SetTable(ItsdbDatabase db) : base(db) { }

	};

	public class ItsdbRunTable : ItsdbTable<ItsdbRun>
	{
		public ItsdbRunTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbRun> _run_id_map = new Dictionary<int, ItsdbRun>();
	};

	public class ItsdbParseTable : ItsdbTable<ItsdbParse>
	{
		public ItsdbParseTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbParse> _parse_id_map = new Dictionary<int, ItsdbParse>();
	};

	public class ItsdbResultTable : ItsdbTable<ItsdbResult>
	{
		public ItsdbResultTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbResult> _result_id_map = new Dictionary<int, ItsdbResult>();
	};

	public class ItsdbRuleTable : ItsdbTable<ItsdbRule>
	{
		public ItsdbRuleTable(ItsdbDatabase db) : base(db) { }

	};

	public class ItsdbOutputTable : ItsdbTable<ItsdbOutput>
	{
		public ItsdbOutputTable(ItsdbDatabase db) : base(db) { }

	};

	public class ItsdbEdgeTable : ItsdbTable<ItsdbEdge>
	{
		public ItsdbEdgeTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbEdge> _e_id_map = new Dictionary<int, ItsdbEdge>();
	};

	public class ItsdbTreeTable : ItsdbTable<ItsdbTree>
	{
		public ItsdbTreeTable(ItsdbDatabase db) : base(db) { }

	};

	public class ItsdbDecisionTable : ItsdbTable<ItsdbDecision>
	{
		public ItsdbDecisionTable(ItsdbDatabase db) : base(db) { }

	};

	public class ItsdbPreferenceTable : ItsdbTable<ItsdbPreference>
	{
		public ItsdbPreferenceTable(ItsdbDatabase db) : base(db) { }

	};

	public class ItsdbUpdateTable : ItsdbTable<ItsdbUpdate>
	{
		public ItsdbUpdateTable(ItsdbDatabase db) : base(db) { }

	};

	public class ItsdbFoldTable : ItsdbTable<ItsdbFold>
	{
		public ItsdbFoldTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbFold> _f_id_map = new Dictionary<int, ItsdbFold>();
	};

	public class ItsdbScoreTable : ItsdbTable<ItsdbScore>
	{
		public ItsdbScoreTable(ItsdbDatabase db) : base(db) { }

		public Dictionary<int, ItsdbScore> _score_id_map = new Dictionary<int, ItsdbScore>();
	};

}