First, we wander over to the AtTask API Documentation and figure out how to authenticate. Notice that every request needs a sessionID - and that sessionID is created by a request to the login url. This means we'll have to make at least two requests - one to nab a sessionID and another to actually get the data that we want.

First, create a new package and drop down all the basic objects we'll need:

1. OLEDB Connection
2. SQL Task: BEGIN TRANSACTION
3. SQL Task: TRUNCATE (truncate target table)
4. Data Flow: Populate table
5. SQL Task: COMMIT TRANSACTION
6. SQL Task: ROLLBACK TRANSACTION (failure predicates from truncate and data flow)

Suggested Control Flow

Because we're trying to maintain a single transaction through multiple executables, set the RetainSameConnection property of your OLEDB connection to True. You certainly could just TRUNCATE and then run the data flow, I just wanted to avoid having the end user(s) hit an empty table mid-load (the API I was working with was a tad slow at times).

Now that you have the control flow under... control... jump in to the Data flow where the interesting stuff happens. You'll need at least a Script Component Source and an OLEDB destination.

Suggested Data Flow

I created two variables to hold the URLs I needed to both login to the API and make the actual request

Open up the Script Component Source and add the variables to the component:

Then configure the output to match what you want to come out of the component:

Finally, edit the script and switch it to build for the .NET Framework 3.5:
Project Explorer > Properties

Select .NET 3.5

Then Add references that we'll need for this script:

Select Assemblies we'll need (System.Runtime.Serialization and System.ServiceModel.Web)

Make sure to Save All at this point! You want to make sure that the changes you made to the project get saved, not just the changes to the main.cs file.

And here's my completed script. The specifics of how you need to login and what the returned data feeds look like will vary, but the basic flow will likely remain the same.

using System;
using System.Data;
using Microsoft.SqlServer.Dts.Pipeline.Wrapper;
using Microsoft.SqlServer.Dts.Runtime.Wrapper;
using System.Xml;
using System.Net;
using System.IO;
using System.Runtime.Serialization.Json;
using System.Runtime.Serialization;
using System.Text.RegularExpressions;
 
[Microsoft.SqlServer.Dts.Pipeline.SSISScriptComponentEntryPointAttribute]
public class ScriptMain : UserComponent
{
    public override void CreateNewOutputRows()
    {
        // loginURL should be a url that returns JSON containing a sessionID
        string loginURL = Variables.loginURL;
 
        // Get the sessionID
        HttpWebRequest request = (HttpWebRequest)WebRequest.Create(loginURL);
        request.Timeout = 300000;
        HttpWebResponse response = (HttpWebResponse)request.GetResponse();
        StreamReader reader = new StreamReader(response.GetResponseStream());
        string json = reader.ReadToEnd();
        Match m = Regex.Match(json, "\"sessionID\": \"(.+?)\"");
        string sessionId = m.Groups[1].Value;
 
        // Need to support pagination, api has a 2000 item limit per req
        XmlNodeList items;
        int rowsPerPage = 1000;
        int page = 0;
 
        // Loop until we find the last page
        do {
            // The taskURL has 3 tokens:
            // {0} = The first record to return for the page we're requesting
            // {1} = The number of records to request
            // {2} = The sessionID
            string taskURL = string.Format(
                Variables.taskURL, 
                page * rowsPerPage + 1, 
                rowsPerPage, 
                sessionId);
            request = (HttpWebRequest)WebRequest.Create(taskURL);
            response = (HttpWebResponse)request.GetResponse();
 
            // Toss the JSON response in to XML so we can work with it better
            XmlDocument xd = new XmlDocument();
            XmlDictionaryReader xr = JsonReaderWriterFactory.CreateJsonReader(
                response.GetResponseStream(), 
                XmlDictionaryReaderQuotas.Max);
            xr.Read();
            xd.LoadXml(xr.ReadOuterXml());
 
            // Put each item returned in to a new Buffer row
            items = xd.DocumentElement.SelectNodes("/root/data/item");
            foreach (XmlNode item in items)
            {
                TaskDataBuffer.AddRow();
                TaskDataBuffer.ProjectId = 
                    item.SelectSingleNode("project/ID").Value;
                TaskDataBuffer.ProjectName = 
                    item.SelectSingleNode("project/name").Value;
 
                // This particular feed allows assignedTo to be empty/null
                if (item.SelectSingleNode("assignedTo/name") != null)
                {
                    TaskDataBuffer.TaskAssignedToName = 
                        item.SelectSingleNode("assignedTo/name").Value;
                }
                else
                {
                    TaskDataBuffer.TaskAssignedToName_IsNull = true;
                }
 
                TaskDataBuffer.TaskId = 
                    item.SelectSingleNode("ID").Value;
                TaskDataBuffer.TaskName = 
                    item.SelectSingleNode("name").Value;
                TaskDataBuffer.TaskPlannedCompletionDate = 
                    item.SelectSingleNode("plannedCompletionDate").Value;
                TaskDataBuffer.TaskPlannedStartDate = 
                    item.SelectSingleNode("plannedStartDate").Value;
                TaskDataBuffer.TaskProgressStatus = 
                    item.SelectSingleNode("progressStatus").Value;
            }
            page++;
        } while (items.Count == rowsPerPage);
    }
}

Remember that each API you interface with will have it's own nuances to deal with and the above code is pretty specific for the AtTask API I was working with. The part I want to point out is after we do all the API busy work, we have that JSON feed in a stream, turn it in to XML, and then access the data using XPath selects.

You would expect to import

That's not what you get, you get an error. I've seen a few different approaches to working around this known issue:

• Replacing embedded text qualifiers with some other token - " in this case
• Import each file as a single column and parse the row in a script task, handling error rows

What I'd like to explore in this post is using regular expressions to transform the entire file in to something that can be natively consumed by SSIS. There are two options for dealing with this:

• Tokenizing the quotation marks found within the cells
• Change the text qualifier

I chose to change the text qualifier because SSIS natively supports multiple character text qualifiers. This will allow me to pick something ridiculous that is incredibly unlikely to appear in the data naturally. Changing the text qualifier also means we won't have to add any extra transformations as we would if we tokenized the embeded quotation marks (to turn them back in to quotation marks).

Let's start out with the file we're importing so we have some text to test against (C:\SSISTextQualifier\Input.csv)

"one","1","two","2","th""r""ee","3","asdf"
"""left","right""","mid""dle","w"",tf","w,""tf",",""omg","omg"","

Now let's iron out the regular expression we're going to need.

We'll start out with grabbing a bunch of stuff between two double-quotes




Ok, we don't just want a bunch of stuff - we want the stuff between each set of quotation marks - we make the * quantifier lazy with a question mark




Better... but that th""r""ee is clearly wrong... we have to stop the final quotation mark from matching properly escaped quotation marks. We can do that with a negative lookahead:




That's different - notice that now properly escaped quotation marks aren't abnormally terminating the column match anymore, now they're just acting like normal characters. We have to make a special exception for escaped quotation marks. Those have to essentially count as a single character:




Perfect - Now we can replace the text qualifiers with something awesome like |~| since we have matched the column content safely

Now that we have our regex, we just have to decide how to execute.

Plan A: SSIS Script Task

Create a new package with a single script task in it

Now edit the script (C#) and add the following up by the existing using statements:

using System.IO;
using System.RegularExpressions;

Modify the Main method to look something like this (depending on your paths):

        public void Main()
        {
            StreamReader sr = new System.IO.StreamReader(@"C:\SSISTextQualifier\Input.csv");
            string allText = sr.ReadToEnd();
            sr.Close();
            StreamWriter sw = new StreamWriter(@"C:\SSISTextQualifier\InputCleaned.csv");
            sw.Write(
                    Regex.Replace(
                        allText, 
                        @"""((?:""""|.)*?)""(?!"")", 
                        @"|~|$1|~|"
                    ).Replace("\"\"", "\"")
            );
            sw.Close();
            Dts.TaskResult = (int)ScriptResults.Success;
        }

Then run that task and you will have a new file called C:\SSISTextQualifier\InputCleaned.csv that's text qualified by |~| and the escaped double quotes have been replaced with a single double quote.

Plan B: Powershell

Since we often call SSIS packages from the SQL Server Agent, we can easily add a Powershell step right before the SSIS package to copy and cleanse our file. If that's the route you choose, your powershell step will look something like this:

gc C:\SSISTextQualifier\Input.csv | 
% {$_ -replace "`"((?:`"`"|.)*?)`"(?!`")", "|~|`$1|~|" -replace "`"`"", "`""} | 
sc C:\SSISTextQualifier\InputCleaned2.csv

Final Steps

However you got your data in to a new file (C:\SSISTextQualifier\InputCleaned.csv in my case) with a new text qualifier, you still have to then import the file. This is done using our old friend the Flat File Connection Manager with the source being the newly minted clean file and with our new text qualifier set as |~|

And that's pretty much it:

Conclusion

Personally, I think the Powershell plan is the cleanest and easiest to implement. The code isn't hidden away in a script task, and it looks cool. Doing it this way allows us to kind of abstract this "file fixing" from the SSIS data flows and let them do what they do best.

The first method is XMLAParallel - in this mode, the application constructs an XMLA command to process the specified number of databases inside of a Parallel element. Only one XMLA command is issued at a time so the applicatoin will not submit another command until the longest processing job completes in each batch.

The second method is Threaded - in this mode, the application creates the specified number of threads and each thread processes databases independantly of the other threads. This allows all threads to remain busy at all times as a given thread will start processing the next unprocessed database as soon as it finishes processing the previous database.

It should be noted that because this app performs only ProcessFull at the database level, it is currently only suitable for processing a large number of small analysis services databases. The environment this was developed for has a process that generates and deploys metadata for over 250 small (< 100MB) ssas databases and we need to process them as quickly as possible.

If you have suggestions for features that would help the application be more useful to you, please post them in the Discussions area.

And yes, I probably should have called it "ParallelDatabaseProcessor." PCP was a cooler acronym.

The application and source code can be downloaded on Codeplex: Parallel Cube Processor

A quick note on what that "Update FTP Connection" task is doing - it's modifying the properties of the "FTP" connection manager to the appropriate Server/Username/Password for this particular client:

1
2
3
4
5
6
7
8
9
10

        public void Main()
        {
            ConnectionManager ftp = Dts.Connections["FTP"];
            ftp.Properties["ServerName"].SetValue(ftp, (string)Dts.Variables["FTPServer"].Value);
            ftp.Properties["ServerUserName"].SetValue(ftp, (string)Dts.Variables["FTPUser"].Value);
            ftp.Properties["ServerPassword"].SetValue(ftp, (string)Dts.Variables["User::FTPPassword"].Value);
 
            Dts.TaskResult = (int)ScriptResults.Success;
 
        }

The problem: Anyone who has tried to upload multiple files to multiple FTP sites in an SSIS package very quickly ran in to FTP errors (that's what happens when you let users tell you what their FTP url and authentication is). You can handle these errors and log them via the standard OnError handler dumping the ErrorCode, ErrorDescription, and usually SourceName out to a flat file or table.

Now, when you look at the ErrorDescription for an FTP Task - you'll be disappointed. It usually states "Unable to connect to FTP server..." or something to that effect. Looking at the execution results in the progress tab, you'll see that the FTP Connection Manager threw a more detailed description of the error:

Good luck getting at that detailed error description - My first attempt here was to acquire a connection and connect via the connection manager and trap the error that came back. While that worked, the error that was trapped was HRESULT 0xC001602A, which basically means that something bad happened.

Now we move on to phase two: Retry the FTP operation via .NET FtpWebRequest and trap that error. This can either flat out replace the FTP Task as a script task or, not sure why I did it this way - I suppose I like to try and use SSIS built in tasks as much as possible - you can retry the operation in the OnError handler assuming you have the requisite information (url, username, and password) available.

Create the OnError handler (either at the package level for everything or just on the executable you care about - I chose the latter):

Then add a data flow task - inside that add a Script Component (as a source) and an appropriate destination (Here it's going to an OleDb destination):

Configure the Script Component for ReadOnly access to the requisite variables:

And configure the Output buffer:

Now go edit the actual script:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

using System;
using System.Data;
using Microsoft.SqlServer.Dts.Pipeline.Wrapper;
using Microsoft.SqlServer.Dts.Runtime.Wrapper;
using System.Net;
using System.IO;
using System.Text;
 
[Microsoft.SqlServer.Dts.Pipeline.SSISScriptComponentEntryPointAttribute]
public class ScriptMain : UserComponent
{
    public override void CreateNewOutputRows()
    {
        string errorDescription = Variables.ErrorDescription;
 
        if (Variables.ErrorDescription.Contains("Unable to connect to FTP server"))
        {
            try
            {
                FtpWebRequest ftp = (FtpWebRequest)WebRequest.Create("ftp://" + Variables.FTPUrl);
                ftp.Method = WebRequestMethods.Ftp.UploadFile;
                ftp.Credentials = new System.Net.NetworkCredential(Variables.FTPUser, Variables.FTPPassword);
 
                StreamReader sourceStream = new StreamReader(Variables.FTPLocalPath);
                byte[] fileContents = Encoding.UTF8.GetBytes(sourceStream.ReadToEnd());
                sourceStream.Close();
                ftp.ContentLength = fileContents.Length;
 
                Stream requestStream = ftp.GetRequestStream();
                requestStream.Write(fileContents, 0, fileContents.Length);
                requestStream.Close();
 
                FtpWebResponse response = (FtpWebResponse)ftp.GetResponse();
 
                response.Close();
 
                errorDescription = "Upload retry attempt via .NET was successful.";
            }
            catch (Exception e)
            {
                errorDescription = e.Message;
            }
        }
 
 
        ErrorSourceBuffer.AddRow();
        ErrorSourceBuffer.ErrorCode = Variables.ErrorCode.ToString();
        ErrorSourceBuffer.ErrorDescription = errorDescription;
        ErrorSourceBuffer.ErrorSource = Variables.SourceName;
        ErrorSourceBuffer.FTPUrl = Variables.FTPUrl;
        ErrorSourceBuffer.FTPUser = Variables.FTPUser;
        ErrorSourceBuffer.FTPPassword = Variables.FTPPassword;
    }
}

And bob's your uncle. Now you have meaningful text in the ErrorDescription column of the output buffer instead of some vague "something's wrong" message.

Here's a flat file destination walkthrough for the OnError handler: http://vsteamsystemcentral.com/cs21/blogs/applied_business_intelligence/archive/2007/04/08/ssis-design-pattern-custom-error-handling.aspx

This is really just an example of the implementation that worked for my purposes - I'm sure some of you would prefer to log elsewhere. This should be enough to get you going. Please drop a line in the Comments if you post alternative approaches.

Usage

SSISForcedLogging.Console.exe "Z:\SSIS Packages"
or
SSISForcedLogging.Console.Exe "Z:\SSIS Packages\Package1.dtsx"

For the following code, be sure to add References to:

• Microsoft.SqlServer.ManagedDTS
• System.configuration

Program.cs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186

using System;
using System.Configuration;
using Microsoft.SqlServer.Dts.Runtime;
using System.Reflection;
using System.Collections;
using System.IO;
using System.Collections.Generic;
 
namespace SSISForcedLogging.Console
{
    class ColumnFilter
    {
        public bool Computer;
        public bool DataBytes;
        public bool ExecutionID;
        public bool MessageText;
        public bool Operator;
        public bool SourceID;
        public bool SourceName;
 
        public DTSEventColumnFilter GetDTSEventColumnFilter()
        {
            DTSEventColumnFilter dtsEventColumnFilter = 
                new DTSEventColumnFilter();
            dtsEventColumnFilter.DataBytes = this.DataBytes;
            dtsEventColumnFilter.Computer = this.Computer;
            dtsEventColumnFilter.ExecutionID = this.ExecutionID;
            dtsEventColumnFilter.MessageText = this.MessageText;
            dtsEventColumnFilter.Operator = this.Operator;
            dtsEventColumnFilter.SourceID = this.SourceID;
            dtsEventColumnFilter.SourceName = this.SourceName;
            return dtsEventColumnFilter;
        }
    }
 
    class Program
    {
        static void Main(string[] args)
        {
            if (args.Length != 1)
            {
                System.Console.WriteLine(
                    "\nPlease specify a package path or directory\n");
                Environment.Exit(-1);
            }
 
            string arg = args[0];
            List<string> packages = new List<string>();
 
            if (File.Exists(arg))
                packages.Add(arg);
            else if (Directory.Exists(arg))
                packages = GetPackagesFromDirectory(arg);
            else
            {
                System.Console.WriteLine(
                    "\nInvalid package path or directory specified\n");
                Environment.Exit(-1);
            }
 
            foreach (string pkgPath in packages)
            {
                System.Console.WriteLine(
                    string.Format("Updating: {0}", pkgPath));
                Application dts = new Application();
                Package pkg = dts.LoadPackage(pkgPath, null);
 
                RemoveOldLogsFromPackage(ref pkg);
                AddLogToPackage(ref pkg);
 
                System.Console.WriteLine(
                    string.Format("Saving: {0}", pkgPath));
                dts.SaveToXml(pkgPath, pkg, null);
            }
        }
 
        public static void AddLogToPackage(ref Package Pkg)
        {
            // Add new provider
            Pkg.LoggingMode = DTSLoggingMode.Enabled;
 
            LogProvider log = Pkg.LogProviders.Add(
                ConfigurationManager.AppSettings["logType"]);
            log.Name = 
                ConfigurationManager.AppSettings["logName"];
            log.Description = 
                ConfigurationManager.AppSettings["logDesc"];
            Pkg.LoggingOptions.SelectedLogProviders.Add(log);
 
            LoggingOptions logOptions = Pkg.LoggingOptions;
            logOptions.EventFilterKind = DTSEventFilterKind.Inclusion;
 
            string[] events = 
                ConfigurationManager
                    .AppSettings["eventsToLog"]
                        .Split(',');
            logOptions.EventFilter = events;
 
            ArrayList columnList = new ArrayList();
            columnList.AddRange(
                ConfigurationManager
                    .AppSettings["columnsToLog"]
                        .Split(','));
 
            ColumnFilter eventColumnFilter = new ColumnFilter();
 
            foreach (FieldInfo field in 
                     eventColumnFilter.GetType().GetFields())
                if (columnList.Contains(field.Name))
                    field.SetValue(eventColumnFilter, true);
 
            logOptions.SetColumnFilter(
                "OnError", 
                eventColumnFilter.GetDTSEventColumnFilter());
 
        }
 
        public static void RemoveOldLogsFromPackage(ref Package Pkg)
        {
            // Remove existing log provider selections
            int count = Pkg.LoggingOptions.SelectedLogProviders.Count;
            for (int i = 0; i < count; i++)
                Pkg.LoggingOptions.SelectedLogProviders.Remove(i);
 
            // Remove existing log providers
            count = Pkg.LogProviders.Count;
            for (int i = 0; i < count; i++)
                Pkg.LogProviders.Remove(i);
 
            RemoveOldLogsFromExecutable(Pkg.Executables);
 
        }
 
        public static void RemoveOldLogsFromExecutable(Executables Execs)
        {
            // reflection ftw
            int count;
            foreach(Executable exec in Execs) 
            {
                foreach (PropertyInfo prop in 
                         exec.GetType().GetProperties())
                {
                    switch(prop.Name) {
                        case "LoggingMode":
                            prop.SetValue(
                                exec, 
                                DTSLoggingMode.UseParentSetting, 
                                null);
                            break;
                        case "LoggingOptions":
                            LoggingOptions opts = 
                                prop.GetValue(
                                    exec, 
                                    null) as LoggingOptions;
 
                            count = opts.SelectedLogProviders.Count;
                            for (int i = 0; i < count; i++)
                                opts.SelectedLogProviders.Remove(i);
                            break;
                        case "Executables":
                            Executables execs = 
                                prop.GetValue(
                                exec, 
                                null) as Executables;
                            RemoveOldLogsFromExecutable(execs);
                            break;
                    }
                }
            }
        }
 
        public static List<string> GetPackagesFromDirectory(string Path)
        {
            List<string> files = new List<string>();
 
            foreach (string d in Directory.GetDirectories(Path))
                foreach (string f in GetPackagesFromDirectory(d))
                    files.Add(f);
 
            foreach (string f in Directory.GetFiles(Path, "*.dtsx"))
                files.Add(f);
 
            return files;
        }
    }
}

App.config

<?xml version="1.0" encoding="utf-8" ?>
<configuration>
	<appSettings>
		<add key="logType" value="DTS.LogProviderEventLog.1"/>
		<add key="logName" value="Windows Event Log Logging Provider (Forced)"/>
		<add key="logDesc" value="Logs Event info to Windows Event Log (Forced)"/>
		<add key="eventsToLog" value="OnError"/>
		<add key="columnsToLog" value="Computer,Operator,SourceName,SourceID,ExecutionID,MessageText"/>
	   </appSettings>
</configuration>