Friday, May 23, 2008

Generate SQL INSERT Statements for a SS2005 Table

For quite a while, I have been looking for a utility that would be able to take a large table stored in SQL Server 2005, and generate INSERT statements. I recently tried a few SQL scripts that I found on the web. Sometimes they worked, sometimes they didn't. But, they all had a problem in the fact that the generated statements were dumped to a SQL Results pane. If you had a very large table, then you had trouble cutting and pasting these statements to your favorite text editor.

I finally got sick and tired of this effort and the limitations that I found. So, in my own spare time over last weekend, I dropped into developer mode and wrote a little C# utility that did what I wanted.

The code below was written very quickly. I didn't use StringBuilder, and I did not try to optimize. This is quick and dirty ... and it works just great.

You can use this code for personal use. You cannot reprint this code anywhere without my permission. Feel free to contact me if you have any enhancements.

Sorry for some of the formatting stuff. In particular, the nice line spacing that I use will not show up on the web page.

-marc







using System;
using System.Data;
using System.Data.SqlClient;
using System.Globalization;
using System.IO;

namespace SQLGenerateInsertStatements
{
/// <summary>
/// This was written by Marc Adler (magmasystems at yahoo dot com).
/// This code can be used freely for your own personal use.
/// It may not be reprinted anywhere without permission of Marc Adler.
/// </summary>
static class Program
{
[STAThread]
static void Main(string[] args)
{
string databaseName = null;
string tableName = null;
string connectionString = "Data Source=(local);Initial Catalog={d};Integrated Security=True";
string outputFile = @"c:\GeneratedStatements.sql";

if (args == null args.Length == 0)
{
Usage();
return;
}

for (int i = 0; i < args.Length; i++)
{
string arg = args[i].ToLower();
if (arg == "-table")
{
tableName = args[++i];
}
else if (arg == "-database")
{
databaseName = args[++i];
}
else if (arg == "-output")
{
outputFile = args[++i];
}
else if (arg == "-connectionstring")
{
connectionString = args[++i];
}
else
{
Usage();
return;
}
}

if (string.IsNullOrEmpty(tableName) string.IsNullOrEmpty(databaseName))
{
Console.WriteLine("The table name or the database name was not specified.\n");
Usage();
return;
}

if (connectionString.Contains("{d}"))
connectionString = connectionString.Replace("{d}", databaseName);

try
{
GetData(databaseName, tableName, connectionString, outputFile);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
}

static private void Usage()
{
Console.WriteLine("SQLGenerateInsertStatements [-help] [-table <tablename>] [-database <databasename>] [-connectionstring <connstring>] [-output <outputfile>]");
Console.WriteLine(@"The default output file is C:\GeneratedStatements.sql");
Console.WriteLine("If the connection string has '{d}' embedded in it, the '{d}' is replaced with the database name.");
Console.WriteLine("If the outputfile string has '{d}' embedded in it, the '{d}' is replaced with today's date.");
}

static private void GetData(string databaseName, string tableName, string connectionString, string outputFile)
{
// Get the (optional) name of the file to write the SQL statements to
if (outputFile.IndexOf("{d}") >= 0)
{
outputFile = outputFile.Replace("{d}", DateTime.Now.ToString("d", new CultureInfo("de-DE")));
}
StreamWriter outputStream = new StreamWriter(outputFile);

// Write the "USE database" statement
outputStream.WriteLine(string.Format("USE [{0}]", databaseName));

using (SqlConnection connection = new SqlConnection(connectionString))
{
using (SqlCommand command = new SqlCommand())
{
// Initialize the SQL Connection
command.Connection = connection;
command.CommandText = "SELECT * FROM " + tableName;
connection.Open();

// Get a DataReader
using (SqlDataReader reader = command.ExecuteReader(CommandBehavior.CloseConnection))
{
// Format the first part of the INSERT statement. This part remains
// constant for every row that is generated.
string sInsert = string.Format("INSERT INTO [{0}] ( ", tableName);
for (int iCol = 0; iCol < reader.FieldCount; iCol++)
{
sInsert += string.Format("[{0}]", reader.GetName(iCol));
if (iCol < reader.FieldCount - 1)
sInsert += ",";
}
sInsert += ") VALUES ({0})";

// Read each row of the table
object[] objs = new object[reader.FieldCount];
int nLines = 1;

while (reader.Read())
{
int n = reader.GetValues(objs);
string sValues = string.Empty;

// Go through each column of the row, and generate a string
for (int i = 0; i < n; i++)
{
try
{
string sVal = (reader.IsDBNull(i)) ? "null" : ObjectToSQLString(objs[i]);
sValues += sVal;
if (i < n - 1)
sValues += ",";
}
catch (DataException)
{
Console.WriteLine(string.Format("Conversion error in Record {0}, Column {1}", nLines, reader.GetName(i)));
return;
}
}

// Dump the INSERT statement to the file
outputStream.WriteLine(string.Format(sInsert, sValues));
nLines++;
}
}
}
}

outputStream.Flush();
outputStream.Close();
}

static string ObjectToSQLString(object o)
{
if (o == null o == DBNull.Value)
return "null";

Type t = o.GetType();

if (t == typeof (string))
return string.Format("'{0}'", ((string) o).Trim());
if (t == typeof (int))
return ((int) o).ToString();
if (t == typeof (long))
return ((long) o).ToString();
if (t == typeof (float))
return ((float) o).ToString();
if (t == typeof (double))
return ((double) o).ToString();
if (t == typeof (bool))
return ((bool) o).ToString();
if (t == typeof (DateTime))
return string.Format("'{0}'", ((DateTime) o));

throw new DataException();
}
}
}


Thursday, May 15, 2008

Vhayu and Aite redux

Our friend Ross Dubin from Vhayu has sent a reassuring message:

Vhayu paid for a research report to be written by Aite. Aite insists on maintaining neutrality in the research and it includes mention of our direct competitors, Kx and OneTick as well as our indirect competitors who handle real-time data analysis, StreamBase, Apama, Aleri, etc. Sang is going to cover the highlights of the research paper during the webcast. That's where his role ends.

Also, our friends from Aleri sent a note to tell me that they are well aware of the lawsuit surrounding Exegy, and that Aleri is well protected from any negative outcomes.

Good news on both fronts.


©2008 Marc Adler - All Rights Reserved

Wednesday, May 14, 2008

Aleri and Exegy

At last week's Accelerating Wall Street conference, it was interesting to see that Aleri brought Scott Parsons from Exegy as their guest.

It was well publicized in the Wall Street press that Hyperfeed had sued Exegy over alleged misdeeds surrounding a botched merger attempt, something that resulted in Hyperfeed going out of business.

To me, a large lawsuit looming over a company like Exegy can bring into question the continued survival of Exegy, should Hyperfeed prevail. I went to Google in order to see if there was any settlement in this lawsuit, and according to the link below, there was an attempt at a settlement, but nothing ever resulted.

http://sec.edgar-online.com/2008/02/29/0000830122-08-000002/Section8.asp

Evidently, Aleri and Exegy have a relationship. However, if I was at a large bank, and I was presented with an Aleri-Exegy solution, I would really have to do some extreme due diligence in the matter. In fact, until this lawsuit was resolved, I would have to be off my rocker to buy anything with Exegy in it.

Exegy is backed by The Acartha Group. I am wondering if Acartha is looking to add a CEP company to their portfolio of companies. I would not be surprised.


©2008 Marc Adler - All Rights Reserved

Analysts and the Vendors They Cover

I received the following invitation the other day:

WEBCAST INVITATION
The Data Management Challenge of Quantitative Analysis

Vhayu invites you to join industry leaders in market data and analytics for a one-hour webcast hosted by Aite Group.

Speakers include:
Sang Lee, Co-founder & Managing Partner / Aite Group

Jeff Hudson, CEO / Vhayu Technologies
David Wilson, Product Manager / Vhayu Technologies

Vhayu has come up with something that they think is an interesting product. Congrats to Jeff, Ross, and the rest of the gang at Vhayu.

However, what is interesting is the fact that they have enlisted Aite group (one of the founders, no less) to host a publicity event.

We look to companies like Gartner, Tabb, and Aite to give us a fairly objective, unbiased look at the technology landscape in the financial sector. However, the presence of an "impartial" analyst and a vendor always gives me an uneasy feeling.

In the dot com era, there was a company/website called Gomez. This company rated the daily effectiveness of online brokers. In fact, CNBC featured The Gomez Report every morning during the go-go years of the late 90's and early 2000's. However, Gomez had banner ads on their website from the brokers who they were supposedly rating.

My opinion is that, if you are going to position yourself as an analyst, you need to give an air of impartiality, and you need to give full disclosure. What is the relationship between Vhayu and Aite? Is Aite receiving any compensation for hosting a webcast that is being created and sponsored by Vhayu? Will Sang Lee agree to host a similar webcast for Vhayu's competitors? Will Sang Lee mention any of Vhayu's competitors during his opening remarks?

For me, there is a bit of a cloud that hangs over any analyst that chooses to associate himself with a particular company.

I like what the Gartner Group does. When I speak at their conference in September, they will not let me speak about, nor will they allow questions about specific vendors. There are no explicit or implicit vendor endorsements allowed when I am on the podium. If a vendor wants to exhibit at a Gartner conference, then they are confined to the vendor area. And, there is no question that Gartner is getting compensated by the vendors for this.




©2008 Marc Adler - All Rights Reserved

Monday, May 12, 2008

Streaming LINQ

Thanks to readers of this blog, here are two open-source implementations of LINQ for streaming data.

Streaming LINQ

Continuous LINQ


©2008 Marc Adler - All Rights Reserved

Wall Street And Tech Conference Report

Jules and I wandered up to the Grand Hyatt around 10:30 for my panel at the Accelerating Wall Street conference that Wall Street And Technology magazine help on May 8th. There were about 80 to 100 registrants at the conference, but I have a feeling that a decent number of those were vendors, since from the second we arrived in the conference, we were accosted by a number of our old friends, like the guys from RTI, Aleri and BEA.

Malcolm West, who is the "Chief Software Architect" of HSBC, never showed up for the panel, so I was on the stage with analysts from Tabb and Aite. All in all, I would say that the panel went very well. The 35 minutes flew by, and I have a feeling that if we had more time, we could have fielded a lot more questions from the audience.

One of the questions that was asked was the possible marriage between distributed memory cache products and CEP products. I have blogged about this here before. I think that there are definite synergies, but the marriage would involve the CEP engine understanding a high-level C# or Java object. And there you have the problems integrating complex objects with the relational structure that is imposed by Streaming SQL. I have a feeling that the integration would be easier with Esper/Nesper.

Another question was about the use of Open Source products. That led me to talk for a few minutes about Esper, and my thoughts on providing an entire eco-system around the CEP engines. However, I get the feeling that Esper is becoming more and more widely used, as it is easy to download and evaluate and get "under the hood" with.

Thanks to the people from WS&T for inviting me ... and a shout out to Ivy Schmerken, whose work I have been reading since I started on Wall Street in 1986. People have come and gone at that publication, but Ivy is the one constant.

The next time I will be speaking is at the Gartner CEP Conference in September. We are taking bets on whether Opher Etzion can stay awake for an entire presentation without playing with his tie.


©2008 Marc Adler - All Rights Reserved

Sunday, May 04, 2008

On Entitlements, Coral8 CCL, SQL, and LINQ

One of the nice things about having kids that are a little older is that it gives me time to putter around on my laptop while watching the Yankees games on TV. I am not doing day-to-day development in Coral8, having handed that aspect of the project over to HH. However, I wanted to see if the entitlements processing of our system could be done in Coral8, which made logical sense.

A brief recap: Our CEP system takes a number of atomic events, puts them through the Coral8 cruncher, and produces derived events. However, we don’t want everyone to see these derived events. We might have information in a derived event that a Prop Trader should not see, or we might have information about a certain financial sector that should be hidden from someone on a trading desk who does not cover that sector.

In addition, we have different kinds of notification mechanisms (GUIs, message buses, email, chat, SMS, etc) that should be utilized depending on the severity level of an event. We don’t want to send several hundred emails to a trader for informational events. However, we might want to email and SMS a trader if we have a “red alert” type of event.

So, we will turn to a familiar pattern called the Recipient List. This is one of the well-documented patterns in the book Enterprise Integration Patterns. I get a good amount of email that asks me for advice on becoming a trading systems developer. My advice is to run, not walk, to this website and book. Most of this stuff is old hack to experienced trading systems developers, but the use cases (especially the one by Jonathan Simon) is worth its weight in gold.

We have come up with a schema and database of entitlement information that marries our users/groups list, severity levels, notification mechanisms, and derived events. As every derived event gets generated by our CEP system, we want to put it through the “entitlements grinder” and come up with a Recipient List of who can see what information in the message, and how they want to be notified of its occurrence.

This seems to be a perfect task for a CEP engine. It can be just one more additional “enrichment filter” whose input we attach to the output of the derived event stream. The output of this enrichment filter consists of the (possibly modified) derived event and the Recipient List.

As an initial step, we implemented the Recipient List Generator as a single SQL query using SQL Server 2005. It is a single query that consists of 2 inner joins and 2 outer joins. It works fairly well.

When I was watching the Yankees game yesterday, I tried porting this query to Coral8. I could not get any variation of this query to compile properly, and when I tried to decompose the query into 4 streams, I got totally different results that what SQL Server gave me. Ideally, “Streaming SQL” languages should be a superset of SQL92. So, in Coral8, if I mirror each SQL Server table as a Coral8 Window with a “KEEP ALL” property, then I should be able to use my SQL Query directly. I would like to do something like this:

INSERT INTO RecipientListOutputStream
SELECT [my original SQL query]

I have given the guys from Coral8 a homework assignment, and asked them to try to take my query and schema and make it work in Coral8.

So, after a frustrating two hours in which I tried to decipher the Coral8 reference documentation and compiler, I decided to turn to another strategy. For shits-and-giggles, I decided to try to write my SQL query in LINQ. I downloaded the experimental Visual LINQ Query Builder from http://code.msdn.microsoft.com/vlinq. I created a new Visual Studio project, pointed the LINQ data sources to the entitlements database, and started plugging away on the VLINQ. In about ten minutes, I had a full LINQ query that implemented my SQL Server query.

(Note: VLINQ was fairly slow on my laptop, and I soon gave up on it, preferring to code the query in LINQ myself. However, Coral8 and other CEP vendors should look at it as a prototype of a visual code generator.)

LINQ has a lot of goodness to it. LINQ is pervasive, and all flavors of LINQ are being developed. I can very well imagine that Microsoft is looking at versions of LINQ that could handle streaming data. Right now, I think that it would be fairly easy to hook up LINQ queries in a pipeline that would handle simple queries on streaming data. Adding streaming SQL constructs is very doable.

If Microsoft was to come out with a Streaming LINQ that is available as part of .NET, how would this affect the world of CEP? An immediate casualty might be NEsper, but that’s OK, since NEsper is just Aaron’s side project right now. But, longer term, I think that a combination of WCF, Streaming LINQ, and a version of Microsoft Analysis Services that was further geared to real-time streams would be a killer to the rest of the CEP industry. (Of course, technology is one thing. Getting all of those Java and Linux bigots over to .Net is another thing.)

©2008 Marc Adler - All Rights Reserved

Friday, May 02, 2008

Tibco EMS and WCF

It looks like Tibco is about to announce WCF support for EMS.

Now, *this* is truly exciting news!

Our framework has an entire hand-written communications layer that allows us to communicate over TCP/IP, EMS, various market data systems, MSMQ, etc. We would like nothing more that to gut this layer and replace it with WCF. The missing ingredient has always been EMS support. Now, hopefully, this will be rectified, and hopefully Tibco will offer it to existing EMS customers at no charge.


©2008 Marc Adler - All Rights Reserved

CEP Forums

We have been asking Coral8 to create a user support forum on their website so that Coral8 users can ask questions of eachother. As long as the various financial institutions don't share their secret sauce, and as long as the discussions are purely technical, then I can imagine that all of the Coral8 users in the various IB's would participate. In particular, I would like to see the sharing of patterns, query optimizations, and adapter functionality.

©2008 Marc Adler - All Rights Reserved

Random Musings

Coral8 just released version 5.3. We asked them for a KDB+ adapter, and they delivered. It was our opinion that KDB+ is used so frequently in capital markets firms that it made perfect sense that the coral8 developer should be able to read data from KDB as easily as they can read data from Oracle or SQL Server. Right now, you still need to write Q queries in Coral8's KDB adapter in order to fetch data from KDB+ .... I was hoping for a way that a developer can write a simple SQL statement and have the KDB adapter translate the query into Q, but that will have to wait for a future version. We ask them to write this stuff and make it available in their core product in the hopes that a lot of people use it, debug it, and ask Coral8 for more enhancements. They did a very basic version, and it will be enhanced per customer demand.

Coral8 also released a Reuters market data adapter, and from what I understand, it will be offered as a separate product that costs a not-too-trivial amount of money. Our internal framework has built-in market data adapters, so we won't be leveraging the Coral8 adapter, especially since it seems like a very vanilla adapter.

It is good that Coral8 has become aware of all of the various adapters needed for firms that do trading. It has been about 6 months since we had to explain what a FIX message was to the Coral8 people, and they have caught on pretty quickly. Coral8 probably had the least amount of captial markets experience of any of the CEP vendors, and they are rapidly catching up.


I read with interest the "exciting announcement" that the banking products side of Aleri were bought by Wall Street Systems. I don't know quite what to make of this. Was this a much-needed infusion of capital? Does Aleri really want to concentrate solely on CEP? Was the banking side of Aleri under-performing? I had lunch yesterday with a vendor of products that are sold into the capital markets space, and the vendor mentioned that the main CEP products that are evaluated are Apama and Streambase, with Coral8 gaining more and more interest. Combined with the difficulty of seeling into capital markets right now, I wonder if this is the first shoe to drop at Aleri.

(A note to PR agencies and Aleri newsletter writers ... you must think that the lives of your readers are pretty drab if you consider the above announcement to be "exciting".... you need to get out of your offices and see what kind of party animals your potential customers are!)

On the plus side, Aleri seems to still be the only company willing to take the STAC challenge. Where are you, Coral8? Apama? Streambase? Esper?


Sprint 2 has completed, and we are starting up Sprint 3. One of the things that is weighing heavily on my mind is the subject of entitlements and Derived Events. Certain users should not see certain derived events at all. Certain users should only see the partial contents of certain derived events.

There is no standard entitlements framework out there. Every IB that I have been with has had multiple custom-built entitlement frameworks. Morgan Stanley had at least 3, and 2 years ago, there was a group that had just been formed in order to build the mother of all entitlement frameworks.

Our entitlements framework needs to work hand-in-hand with our message bus. Most CEP applications are fairly simplistic, and their output goes into a single system, so there is no need for entitlements. Other CEP applications want to publish everything out to everyone ... a surveillance-type CEP application might be crippled if its output is only going to the security guard who is in the middle of a doughnut break. We can't let the prop traders see agency flow, and vice-versa. We can't let certain people on a single desk see what others on the desk are doing ... but the head of the desk should be able to see everything, and if the head of the desk is on vacation, the notifications should be transmitted to a chain of proxies.

It would be great if this kind of feature was built into a CEP engine ... given a derived event and a list of fine-grained entitlements, produce a "recipient list" of what message bus topics we send the derived event to.





©2008 Marc Adler - All Rights Reserved

Tuesday, April 29, 2008

We are hiring again

Even though you may be reading about layoffs on Wall Street, we still have openings in Equities IT for developers and technologists who are very smart and are passionate about technology. Business experience preferred (Risk, Trading, Analytics, CEP). The positions can be in New York City, in Jersey City, or in Warren, NJ. Java or .NET.

I still have openings for a great UI developer (we are moving to WPF) who has experience with real-time systems, and a more "analytical" person who can analyze flowing equities trading and risk data in real-time and come up with interesting trading decisions.

Email me if you are interested.

©2008 Marc Adler - All Rights Reserved

I will be at the Accelerating Wall Street conference on May 8

I just got invoted to particpate on a panel about CEP. It's goign to be on Thursday, May 8th at the Grand Hyatt in NYC. The conference is titled "Accelerating Wall Street", and is being sponsored by Wall Street and Technology Magazine.

Here is a link to the conference:

http://wallstreetandtech.com/accelerate/agenda.jhtml;jsessionid=RR5ALTPMXDVHYQSNDLOSKH0CJUNN2JVN

My session is "CEP on Wall Street". Here is the blurb:

Complex Event Processing (CEP) holds the promise of major benefits for Wall Street firms, as it can speed up the interpretation of volumes of data in real time. Where is CEP being used today? Where will it be used in the future? Is it more hype than reality? In this special Get to the Point session, panelists will have 60 seconds to answer questions for the moderator and the audience.

Moderator:
Greg MacSweeney, Editor-in-Chief, Wall Street & Technology

Panelists:
Malcolm West, Chief Software Architect, HSBC's Corporate, Investment Banking and Markets division
Adam Honoré, Senior Analyst, Aite Group, LLC

©2008 Marc Adler - All Rights Reserved

Friday, April 18, 2008

Abstracting the CEP Engine

Here are two comments that I received yesterday, and my answers:

1) You bought an CEP Engine that doesn't support event clouds?

We feel that Coral8 does support event clouds, but we are looking for the best pattern to implement it. Mark, who is the CTO of Coral8, doesn't quite agree with the term "event cloud". His posting here highlights his argument. According to Mark, an "event cloud" can be represented as multiple event streams, something that Coral8 supports.


2) How and why did you abstract the CEP engine in your system?


First, the why. We want to insulate ourselves from any uncertainties concern the CEP engine, both in terms of the product itself and of the company. In this economic environment, we are concerned that some of these smallish CEP companies might be strained. Ones who are backed by Venture Capital might find their VC's getting worried and thinking that we are reliving those inglorious times from 2002 to 2003. Ones who are privately financed might find that the backers want to move into other areas. It is no secret that most firms are cutting back or delaying their software purchases, and the ones who get impacted first are the smaller niche companies.

We also want to have some flexibility in case the CEP engine itself does not function as advertised. Coral8 has given us great support, but we have not stressed it yet. We know other companies who have evaluated Coral8 who have foudn some shortcomings, things that the Coral8 staff have addressed. However, it is perfectly within the realm of possibility that we may need to consider another CEP engine should Coral8 fall on its face.

Now, the how ....

We are not using Coral8's native input and output adapters. We are not even reading databases using Coral8's PollFromDatabase and ReadFromDatabase adapters. We have an input server that is used to read static and real-time data and marshall that data into coral8 tuples. On the other side, we have an output server that takes the derived event tuples from Coral8, marshalls them into a common format, and does various kinds of alerting and visualizations.
From the days that we did evaluations of other CEP vendors, we have layers in our input and output servers that deal with Aleri and Streambase. In other words, we have our own adapters! Changing from Coral8 to Streambase or Aleri involves a simple edit to our Spring-like configuration files.

In addition, we have the ability to farm out work to other engines, such as KDB+. We can then read the derived events that are generated by other systems (as long as they are in our common format) and put them into the CEP engine's "event cloud".

In our architecture, we have introduced extra hops in order to abstract the CEP engine. But, we are not that concerned, since we are dealing with analysis and alerting rather than trading.

©2008 Marc Adler - All Rights Reserved

Wednesday, April 16, 2008

Aleri and STAC

Congrats to Aleri for being the first CEP vendor to volunteer their product to undergo scrutiny from STAC Research.

STAC first announced this program at last September's Gartner CEP Summit, but I am surprised that none of the other CEP vendors have joined this program. Aleri's participation means one of two things ---- they are very confident in the power of their CEP engine and/or they are willing to pony up the participation fee.

I am interested to see the test cases that STAC comes up with. Hopefully, these test cases will be created by an impartial party.


©2008 Marc Adler - All Rights Reserved

Sunday, April 13, 2008

More Cloudy Thoughts

It seems that my inelegance in describing my idea of the event cloud has spurred some debate between Greg and Hans. I am going to try to clarify my thoughts around the event cloud, and see if it makes more sense.

I don’t want to give any of our “secret sauce” away in my postings, so I will try to map my thinking from the Equities Trading domain to another domain that I am not familiar with …. The domain of building security. (Apologies in advance to all of the sleuths who are reading this. My use cases will probably malign your esteemed field of study.)

Let’s say that we are writing an enterprise-wide CEP application for our MegaBank that will monitor our security system so that those nosey parkers from StanLehGoldBar Inc don’t break into our building and steal our secrets.

We are going to make the distinction between Atomic Events and Derived Events. A Derived Event is generated when something interesting is detected from the monitoring of one or more atomic events.

An atomic event may be saved in our CEP engine, depending on the use case. A derived event will be definitely be saved by our CEP system. Since a derived event represents an “interesting” condition, we may want to do reporting and analysis on derived events. We may also want to do some sophisticated searching of our derived events. In my mind, the persisted atomic and derived events form our event cloud.

Two or more derived events can be combined to form a new derived event. A derived event can be combined with an atomic event to form a new derived event. Let me illustrate this.

We have a lot of atomic events that come through our CEP system. Every person that passes through MegaBank’s doors generates a PersonEnteredBuilding atomic event. (This is analogous to a market data event.) We have a static database of all of MegaBank’s employees, and when the person in a PersonEnteredBuilding event does not match an entry in our employee database, the CEP system generates a NonEmployeeEntered derived event. This derived event will be persisted in our CEP engine.

We also have a derived PersonLeftBuilding event generated whenever a person leaves the building. This event is generated

(Let’s fantasize a bit, and assume we do a retina scan of everyone entering the building. Let’s stretch our imaginations a bit and say that we have a retina scan of all StanLehGoldBar Inc employees)

Let’s say that we do a join between a NonEmployeeEntered event and the static source of StanLehGoldBar employees. We can generate a CompetitorInBuilding derived event. This derived event will be saved in the CEP engine too.

We monitor all doors to our building. Whenever someone goes through the doors of the Equities trading floor, we generate a TradingFloorEntered atomic event. If someone goes through the trading floor doors for the first time of the day after 6:00 PM, we might want to consider that action to be a suspicious activity, and we want to generate a PossibleIntruderOnTradingFloor derived event.

Now, some smart security guard who is a user of our CEP system wants to create a new derived event. He wants to see if we have a competitor roaming around our Trading Floor at night. So, the security guard goes into our CEP GUI and creates a new, custom derived event called CompetitorOnTradingFloorAfterHours that gets generated 1) if we have a CompetitorInBuilding derived event 2) that does not have a “cancelling” PersonLeftBuilding event, 3) combined with a PossibleIntruderOnTradingFloor event. The fact that various events are “floating” around our event cloud makes it possible to easily create new derived events on the fly.

The security guard might also want to do a query of our event cloud to see if the competitor was “casing the joint” prior to the intrusion into the trading floor at night. The security guard might want to see how many times that particular competitor visited MegaBank over the past month. He might want to see how many times the competitor visited the trading floor during normal business hours. This sounds like the kind of query that you would do with a Data Warehouse, not a CEP engine.

So, we need to be able to represent the event cloud in an efficient way in the CEP engine. We need to be able to create new derived events dynamically, while the CEP engine is running. We need to be able to do ad-hoc analysis of the event cloud to improve our situational awareness. We need to be able to create new and interesting visualizations that let the user peek into what is going on inside the cloud.

Interesting stuff, right?

Tim Bass gives a list of techniques that can be used to perform analysis of events. These techniques include:


  • Rule-Based Inference
  • Bayesian Belief Networks (Bayes Nets)
  • Dempster-Shafer’s Method
  • Adaptive Neural Networks
  • Cluster Analysis
  • State-Vector Estimation
I have to admit that I have not delved into these areas before. However, we just hired someone with a BioInformatics background who we hope could do this stuff in his sleep. Who would have thought that, working for MegaBank, I would be exposed to such interesting areas of study?


©2008 Marc Adler - All Rights Reserved

How to do an UPSERT in Coral8

Assume that the LastTrades window has the retention policy of KEEP LAST PER Symbol.

Here is some code (provided by Mark of Coral8) to do an upsert.

INSERT INTO
LastTrades
SELECT
SPC.symbol,
SPC.price,
If LT.volume Is Null Then 0 Else LT.volume End If
FROM
StreamPriceCorrections SPC
Left Outer Join LastTrades LT ON SPC.Symbol = LT.Symbol;


If we need to do an update rather than an upsert, this piece of code works:


INSERT INTO
LastTrades
SELECT
SPC.symbol,
SPC.price,
LT.volume
FROM
StreamPriceCorrections SPC, LastTrades LT
ON SPC.Symbol = LT.Symbol;



©2008 Marc Adler - All Rights Reserved

Addendum - Cloudy Thinking

In my previous post, I don't think that I stated my question properly.

I was asking how to actually implement the Event Cloud in a Streaming SQL-like CEP engine, so that we can build up hierarchies of events. I was looking for schemas, SQL statements, best practices, etc.

I am well aware of POSETS, but what might be easy to implement in C# or Java-based data structures is not that easy in a quasi-relational system such as CORAL8.

So, I was hunting for advice from people who might have implemented event clouds in Coral8, Streambase, and Aleri, all three which are based on SQL.

I will follow up this post with a more concrete example.

©2008 Marc Adler - All Rights Reserved

Saturday, April 12, 2008

Cloudy Thinking

One of the things that has been on my mind recently is how to represent the "event cloud".

One of the buzzphrases that comes out of the CEP movement is the term "event cloud". This is the huge amalgamation of all of the events that flows through your event processing system.

This week, I had the opportunity to talk with Mary Knox of the Gartner Group. Mary follows the world of CEP. Just like many of the executives at the various CEP companies have told me, Mary confirms that many companies in finance are using CEP for simple streams of processing .... a small algo trading system here, a pricing engine there, a portfolio analysis system in another place. Most shops seem to be "under-utilizing" their CEP engines. Our effort is probably one of the few out there that will really attempt to poke holes in the hype surrounding CEP.... either our application will validate the hype, or all of the CEP engines will flame out in a blaze of glory.

Mary has not heard of too many financial companies who are implementing the event cloud, so I wonder if we are breaking new ground in that area. We know what an event cloud is and what it is supposed to do, but how do we actually implement the event cloud in a system like Coral8 or Streambase? Are relational tables/windows the best way to represent the cloud? And, what about hierarchies of events that are derived from other events? How can be best represent that complex graph of inter-relationships with SQL-ish windows?

Is anyone out there doing similar work with the cloud?


©2008 Marc Adler - All Rights Reserved

Sunday, April 06, 2008

Coral8 is Our Choice or “How the hell did we get here?”

When we went down to Orlando last fall to attend the Gartner Summit on Complex Event Processing, we went with eyes wide open. We were new to the domain of CEP, and one our missions was to try to pick a vendor for the CEP engine that would drive our efforts to produce a major CEP system for our Equities business.

There were a bunch of event-processing systems that were not under consideration because it seemed that they had moved into the strictly vertical area of Algo Trading. These CEP systems included Truviso and Skylar. We needed a general-purpose CEP system, and we wanted to only consider systems that still had a generalist product. An exception to this rule was Aleri, a company who had just come out with a Liquidity Management System as a separate product. We thought that Aleri would still keep its focus on the core CEP engine, so it warranted inclusion of our evaluation.

Apama fell into the Algo trading vertical, but Apama still has a general purpose CEP engine. However, when we tried to evaluate Apama, we were told that we had to go through the dog-and-pony marketing show, something that we did not want to do. I am not sure if this requirement was brought on by the purchase of Apama by Progress Software, a company who I think of as being Computer Associates Lite. I was also told about some interesting experiences between Apama and a major bank by a former colleague of mine whose opinion I trust, and this also influenced by decision to evaluate Apama. This was unfortunate, as I happen to side more with Apama on the whole EPL vs Stream SQL debate.

This left four systems: Streambase, Coral8, Esper, and Aleri.

Coral8 had always been the front-runner, mostly due to recommendations by some former colleagues who were at Merrill Lynch. I had always liked the “vibe” surrounding Coral8, and their openness at giving out eval copies of their software.

Readers of my blog know that I had strong negative opinions about Streambase because of the aggressiveness of their marketing department, an opinion which was shared by a lot of people out there. Nevertheless, their new CEO, Chris Risley, contacted me personally and told me that he had addressed my concerns. After Chris and Richard Tibbetts came down to NYC to meet with me, we decided to include Streambase in the evaluation.

I really wanted to try Esper, but there were a few things that worked against them. The primary factor was that the .NET version, NEsper, was something that was developed by the hard-working Aaron Crackajaxx for his business needs, and did not seem to be part of the mainline Esper product line. We are a .NET shop here, and we needed a product that supported .NET as a first-class citizen. If Aaron decided to become disinterested in Nesper, or if he moved on to another company, then where would we be? We also preferred a product that had an entire ecosystem built around it. So, we passed on Esper. However, Esper is still very much on my radar screen, and I am interested to see how Thomas continues to develop the company and the product.

We spent a good deal of time evaluating Aleri, and most of these experiences were detailed in past entries in this blog. We really wanted to see Aleri succeed, as they were a local company, staffed with a lot of very smart and gentile ex Bell Lab-ers. However, we felt that their product was not ready for us, mostly because of what I called the “spit and polish” issues. I won’t rehash the details now, but if you are interested, please go back and read the old entries in this blog. The areas that needed improvement in the Aleri product were the Aleri Studio, the documentation, and the integration of external data sources.

I met a good deal with Don DeLoach, the CEO of Aleri, and the one positive that will come from my rejection of Aleri is a renewed focus by Aleri on the aesthetics of their product. You can already see these efforts by reading the new Aleri Blog. From what Don had told me a few months ago, their 3.0 product will start to focus on easier integration of external data sources, and will have much improved documentation. I look forward to seeing their efforts come into fruition.

Coral8 was always the front-runner in our evaluation. Their engine is written in C++. They had a decent .NET SDK that let you build out-of-process adapters in C#, and also let you interact with the internals of the Coral8 engine. Their documentation was good, although a bit obtuse at times, and the documentation was backed up by a ton of whitepapers that are available on their website. Their Coral8 Studio gives you a source code view of development, and the GUI part of the Studio is updated after every compile of the source. The CEO of the company was the person who created Crystal Reports, and knows what it takes to build a software company. But, most of all, their CTO and President interacted with us all of the time, and was extremely receptive to our ideas on improving his product. I like when a CTO and the pre-sales engineer send me mail on a Sunday morning!

Streambase was a strong contender. They had some great features in the product that Coral8 has only just come out with (ie: windows that are bucketed by column value). Their GUI is very strong, and their documentation and tutorials are first class. However, I have to say that the interest that Coral8 has shown in our success, and the availability of their CTO was what tipped the odds in Coral8’s favor.

By now, you must be saying to yourself “Where’s the meat?” Didn’t we try to soak and stress the various engines? Didn’t we have an OPRA feed running into the engines in an effort to break them? Didn’t we monitor the use of the CPU and other computing resources? Well … no. To tell you the truth, we were relying on STAC Research to try to do that job for us. STAC is only now just starting to get up to speed in the CEP world, and we will be monitoring their efforts in this space. The general feeling is that most of these CEP engines perform in roughly the same manner, and if one of the CEP engines is 5% faster than Coral8, it is not going to sway our decision, since we are not that concerned right now with super low latency. We are more concerned with the intangibles; responsiveness of the support organization, evolution of the product (and our input into the roadmap), support for .NET as a first-class citizen, stability of the company, etc.

Despite choosing Coral8, we have been careful in our architecture to abstract the specific CEP engine, and no external system will know that Coral8 is driving our CEP system. In the same way that CEP engines have abstracted datasources by using pluggable adapters, we have abstracted the CEP engine. Yes, we have chosen Coral8, and so far, we are satisfied by our choice. But, we are also keeping our eyes open for how the other products evolve in the world of Complex Event Processing.



©2008 Marc Adler - All Rights Reserved

Of Webcasts, Vendors, and Perceptions

Some people have pinged me to find out why I never appeared on the webcast with Coral8 a few weeks ago. Basically, it was because of a mess-up by the folks at Incisive Media.

You had to call a certain phone number to participate in the webcast. The operator dutifully asked me for my name and my company. I was then transferred to what turned out to be a listen-only line which enabled me to hear the speakers, but prevented them from hearing me. After the presenter from The Aite Group gave his spiel, the presenters asked if I was on the line. From my perch on the trading floor, I screamed into the phone “I’m here! I’m here!” … but since I was on a listen-only line, nobody on the other side could hear me. I furiously sent emails to the guys at Coral8 and Incisive Media, but all were oblivious. I finally hung up, called back the operators, and explained that I was one of the presenters, and the operator put me on the presenters-only line, but by that time, John Morrell had started his 30-minute pitch for Coral8. I had another meeting scheduled for noon, so I had to leave the webcast.

I have to admit that I had no idea that I was going to be part of a Coral8 infomercial, and if I did, I would not have been part of the webcast. As a good member of MegaBank, I need to be extremely careful about the PERCEPTION of a relationship between a specific vendor and myself. In fact, recently, I felt that I had to cancel the recording of a CEP podcast when I found out that the people who set up the podcast were bringing the Chief Architect of BEA Systems to the interview to ask questions. Not only do I not want to be perceived as a shill for a specific vendor, but I do not want my name on a banner next to the name of a company whose products I am not using. Even if the BEA architect’s questions to me were vendor-neutral, having my name on a podcast that has the BEA name on it as the sponsor would imperceptibly tie me into BEA.

The SOA/Web Services conference that I spoke at a few weeks ago had no vendor tie-in. I was on a panel with someone from Bank of America and from Google. There were no SOA/WebServices vendors on the panel. I am also scheduled to speak about CEP at a big conference in a few months, and I will be careful not to mention the specific technology stack that we are using. I don’t mind being mentioned as a reference customer for a vendor, under the conditions that

- I am actually using the product
- I am completely satisfied with the product
- My current employer gives me permission

Being a member of the management layer at MegaBank and blogging at the same time is a tricky proposition. If you read something here, you need to be thoroughly convinced that I am not shilling for any vendor. I disparage all vendors equally. I am not like certain consulting firms, who have relationships with every vendor and will never render a negative opinion about any of those vendors IN PUBLIC.

©2008 Marc Adler - All Rights Reserved