I recently had a need to do a deep dive into SSIS DTS packages to import customer data from a MVC WEB API site. While I have done this many times via a windows services or other such similar code, I really wanted to manage this process via SQL Server jobs and data transformation services.

After experimenting and looking about the web for more information on this I quickly determined it was fairly easy to accomplish. I will post some demo code on how I did this… Hope this helps someone!

I know I needed to store a list of customers and their unique URLs for each WEB API end point for retrieving data. My specific business domain is in education so I knew I would be calling data via district, school, students and teachers by schools, schedules by schools and etc… To help manage this I created a simple database that tracks the various data points I need. This database is nothing more than a dumping ground for the API data and uses varchar to hold most of the data before I parse it into our actual production systems. Each time the SSIS package runs I truncate the tables, which I recommend to reduce log file entries.

The first table I created in my SSISImport database was my customer table. This table is used to store a list of customers who use this MVC API and their security certs as needed.

Table Name: Customer

The Cert field for our implementation is unique to each customer and I recommend in your production system to obfuscate this information via encryption.

Then I created a table to hold each customer’s API URL formatted to allow string.format (c#) to insert the school and student into the url.

For example when I need a list of students for a particular school my API URL looks like this in the database: https://www.schooldistrict.us/ssisapi/api/schools/{0}/students

The {0} parameter is replaced with a value in my code as it iterates over each school.

Table Name: APIUrl

Entries in this table looks like this:

This table is called in the code for each customer by each DTS Task that is belongs to and etc.

The rest of the tables in the database are nothing more than copies of the POCO object format in a database structure so I can desterilize it from json into a flat table in sql server. You will see this in the c# code coming up next.

The first DTS task sets are nothing more than the truncate code:

EXECUTE SQL TASK…

The task (Data Flow Task – Get Customers) grabs all of the customers and stores them in a variable being passed to the DTS Foreach loop task.

The foreach look task then loops over each customer and calls their APIs one by one until the operation is complete…

Now the meat of the code… I have simplified this example for this demonstration. I would recommend adding in more error trapping and exception handling…

Unfortunately, LinkedIn Blog does not have a nice code format tool…

<code>

/* Microsoft SQL Server Integration Services Script Component

* Write scripts using Microsoft Visual C# 2008.

* ScriptMain is the entry point class of the script.*/

using System;

using System.Data;

using Microsoft.SqlServer.Dts.Pipeline.Wrapper;

using Microsoft.SqlServer.Dts.Runtime.Wrapper;

using System.Data.SqlClient;

using SC_5baebd7e1dfe4a73abb38025972f5ed7.csproj;

using System.Collections.Generic;

using SC_5baebd7e1dfe4a73abb38025972f5ed7.csproj.Models;

using System.Net;

using System.IO;

using System.Web.Script.Serialization;

using System.Windows.Forms;

using Microsoft.SqlServer.Dts.Runtime;

using System.Collections;

[Microsoft.SqlServer.Dts.Pipeline.SSISScriptComponentEntryPointAttribute]

public class ScriptMain : UserComponent

{

public override void PreExecute()

{

base.PreExecute();

/*

Add your code here for preprocessing or remove if not needed

*/

}

public override void PostExecute()

{

base.PostExecute();

/*

Add your code here for postprocessing or remove if not needed

You can set read/write variables here, for example:

Variables.MyIntVar = 100

*/

}

public override void CreateNewOutputRows()

{

/*

Add rows by calling the AddRow method on the member variable named “<Output Name>Buffer”.

For example, call MyOutputBuffer.AddRow() if your output was named “MyOutput”.

*/

ProcessData();

}

private void ProcessData()

{

try

{

var myConnectionString = this.Connections.StudentInfoManager.ConnectionString;

using (SqlConnection conn = new SqlConnection(this.Connections.StudentInfoManager.ConnectionString))

{

if (conn.State != ConnectionState.Open) conn.Open();

using (SqlCommand command = conn.CreateCommand())

{

int customerId = this.Variables.CustomerID;

string Cert = this.Variable.Cert;

var customerUrl = GetCustomerWebServiceUrls(command, customerId);

// Recommend error trap

var CourseList = GetWebServiceResult(customerUrl.WebServiceUrl, Cert);

// Recommend error trap

if (CourseList.Length > 0) // possible null exception

{

foreach (Courses item in CourseList)

{

Output0Buffer.AddRow();

Output0Buffer.CustomerID = item.CustomerId;

Output0Buffer.LongDescription = item.LongDescription;

Output0Buffer.StateCourseCode = item.StateCourseCode;

Output0Buffer.Title = item.Title;

}

}

}

if (conn.State != ConnectionState.Closed) conn.Close();

}

}

catch (Exception ex)

{

FailComponent(“ProcessData”, “Error: ” + ex.Message);

}

}

private APIUrl GetCustomerWebServiceUrls(SqlCommand command, int customerId)

{

APIUrl apiURL = new APIUrl();

try

{

command.CommandText = String.Format(“Select * from APIUrl Where customerID = {0} And APIType = ‘Courses’ Order By CallOrder”, customerId);

command.CommandType = CommandType.Text;

using (SqlDataReader reader = command.ExecuteReader())

{

if (!reader.HasRows)

{

throw new NoRecordFoundException(String.Format(“Table APIUrl for customer {0} does not have a Courses entry.”, customerId));

}

int rowsInDataReader = 0; //Should always be 1

while (reader.Read())

{

rowsInDataReader++;

//Cast to APIUrl object

apiURL = new APIUrl()

{

Id = (int)reader.GetValue(0),

customerId = (int)reader.GetValue(1),

APIType = (string)reader.GetValue(2),

WebServiceUrl = (string)reader.GetValue(3),

CallOrder = (int)reader.GetValue(5)

};

}

if (rowsInDataReader != 1)

{

FailComponent(“GetCustomerWebServiceUrl”, String.Format(“More than one URL entry was found in the database for Customer: {0} See table APIUrl”, customerId));

return null;

}

}

}

catch (Exception ex)

{

FailComponent(“GetCusotmerWebServiceUrl”, “Error: ” + ex.Message);

}

return apiURL;

}

private Courses[] GetWebServiceResult(string WebServiceUrl, string Cert)

{

string wsUrl = WebServiceUrl;

//MessageBox.Show(wsUrl);

Courses[] jsonResponse = null;

try

{

HttpWebRequest httpWReq = (HttpWebRequest)WebRequest.Create(wsUrl);

httpWReq.Headers.Add(“CERT”, Cert);

HttpWebResponse httpWResp;

httpWReq.MaximumResponseHeadersLength = 10000;

httpWResp = (HttpWebResponse)httpWReq.GetResponse();

//MessageBox.Show(“Waiting for response”);

//Test the connection

if (httpWResp.StatusCode == HttpStatusCode.OK)

{

//MessageBox.Show(“Waiting for response : OK”);

Stream responseStream = httpWResp.GetResponseStream();

string jsonString = null;

//Set jsonString using a stream reader

using (StreamReader reader = new StreamReader(responseStream))

{

//MessageBox.Show(“Stream reader”);

jsonString = reader.ReadToEnd();

reader.Close();

}

//MessageBox.Show(“WStream read”);

//Deserialize our JSON

JavaScriptSerializer sr = new JavaScriptSerializer();

sr.MaxJsonLength = Int32.MaxValue;

jsonResponse = sr.Deserialize<Courses[]>(jsonString);

}

else

{

string errorMessage = “Web service response Failed with Status Code: ” + httpWResp.StatusCode;

FailComponent(“GetWebSerbiceResult”, “Web Service Url: ” + wsUrl + ” ” + errorMessage);

//MessageBox.Show(errorMessage);

}

}

catch (WebException ex)

{

//MessageBox.Show(“WebException ” + ex.ToString());

FailComponent(“GetWebServiceResult”, “WebException: ” + ex.Message);

}

catch (Exception ex)

{

//MessageBox.Show(“Generic EX ” + ex.ToString());

FailComponent(“GetWebServiceResult”, “General Exception: ” + ex.Message);

}

return jsonResponse;

}

private void FailComponent(string SubComponent, string message)

{

bool fail = false;

IDTSComponentMetaData100 compMetadata = this.ComponentMetaData;

compMetadata.FireError(1, SubComponent, message, “”, 0, out fail);

}

}

public class NoRecordFoundException : Exception

{

public NoRecordFoundException() : base() { }

public NoRecordFoundException(string message) : base(message) { }

public NoRecordFoundException(string message, Exception innerException) : base(message, innerException) { }

}

</code>