Getting started with the AWS JavaScript SDK#

A desire to search through sondehub-history on S3 for https://sondehub.org motivated me to learn about the AWS JavaScript SDK. These are notes that I took while exploring the SDK and historic data.

There’s an excellent talk about radiosondes and sondehub here: STM32 Development Boards (literally) Falling From The Sky.

Searching through sondehub-history on S3#

Region and bucket were fetched from https://sondehub-history.s3.amazonaws.com/index.html#launchsites/

Region: us-east-1
Bucket: sondehub-history

AWSJavaScriptSDK version fetched from https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/

Include <script src="https://sdk.amazonaws.com/js/aws-sdk-2.1029.0.min.js"></script>

Configure and create your AWS.S3 instance:

AWS.config.region = 'us-east-1';
 
var s3 = new AWS.S3({
  params: {Bucket: 'sondehub-history'}
});

And now you’re ready to start searching!

Alternatively, go to https://sondehub-history.s3.amazonaws.com/index.html#launchsites/ and hit F12 to bring up the developer console and you can start playing with s3.

s3.listObjectsV2() allows you to fetch all objects, 1000 at a time, by checking if IsTruncated is true and then by providing the NextContinuationToken as described in https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjectsV2-property

But…

You’ll get a CredentialsError: Missing credentials in config error if you call s3.listObjectsV2() directly without credentials.

s3.makeUnauthenticatedRequest()#

You don’t need credentials to search this public S3 bucket.

You can call s3.makeUnauthenticatedRequest('listObjectsV2', params, function(err, data) {

Filtering for objects with params#

params for listObjectsV2() has Delimiter to filter by the launchSiteId, and a Prefix to filter by the date.

params doesn’t accept a Suffix (sondeId) to filter on.

var params = {
  Bucket: 'sondehub-history',
  Delimiter: 'launchsites/' + parseInt(launchSiteId) + '/',
  Prefix: 'date/' + datePrefix
};

All the content can be pulled back recursively and then the Key can be filtered on by Suffix (sondeId):

data.Contents.filter(item => item.Key.endsWith(sondeId + '.json')).forEach(element => console.log(element.Key));

Example output#

Fetching...
date/2021/11/15/T3750254.json
date/2021/11/16/T3750254.json
Done!

Find folders available for a particular launch site by using Delimiter#

https://sondehub-history.s3.amazonaws.com/?prefix=launchsites/95527/2021/&delimiter=/

<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>sondehub-history</Name>
<Prefix>launchsites/95527/2021/</Prefix>
<Marker></Marker>
<MaxKeys>1000</MaxKeys>
<Delimiter>/</Delimiter>
<IsTruncated>false</IsTruncated>
<CommonPrefixes><Prefix>launchsites/95527/2021/09/</Prefix></CommonPrefixes>
<CommonPrefixes><Prefix>launchsites/95527/2021/10/</Prefix></CommonPrefixes>
<CommonPrefixes><Prefix>launchsites/95527/2021/11/</Prefix></CommonPrefixes>
<CommonPrefixes><Prefix>launchsites/95527/2021/12/</Prefix></CommonPrefixes>
</ListBucketResult>

HTML listLaunchSiteFilesForSonde.html


<!DOCTYPE html>
<html>
  <head>
  <script src="https://sdk.amazonaws.com/js/aws-sdk-2.1029.0.min.js"></script>
  <script>
  AWS.config.region = 'us-east-1';
  
  var s3 = new AWS.S3({
    params: {Bucket: 'sondehub-history'}
  });

  function listLaunchSiteFilesForSonde(launchSiteId, sondeId, datePrefix, getAll, token) {
  
    var params = {
      Bucket: 'sondehub-history',
      Delimiter: 'launchsites/' + parseInt(launchSiteId) + '/',
      Prefix: 'date/' + datePrefix
    };
  
    if (typeof token !== 'undefined') {
      params.ContinuationToken = token;
    }
  
    s3.makeUnauthenticatedRequest('listObjectsV2', params, function(err, data) {
      if (err) {
        console.log(err, err.stack);
      } else {
        // Filter and display as we go instead:
        data.Contents.filter(item => item.Key.endsWith(sondeId + '.json')).forEach(element => console.log(element.Key));
  
        if (data.IsTruncated == true && getAll == true) {
          listLaunchSiteFilesForSonde(launchSiteId, sondeId, datePrefix, getAll, data.NextContinuationToken);
        }
  
        if (data.IsTruncated == false) {
          console.log('Done!');
        }
      }
    });
  }
  
  // launchSiteId = 95527
  //      sondeId = 'T3750254'
  //   datePrefix = '2021/11/16' etc
  //       getAll = true|false (i.e. get all 117,007+ results)
  //        token = NextContinuationToken if IsTruncated is true and getAll is true

 </script>
 </head>
 <body>
   <h1>Hit F12 and check the console! :-P</h1>
   <input type="button" onclick="listLaunchSiteFilesForSonde(95527, 'T3750254', '2021/11', true);" value="Search"/>
  </body>
</html>