############################################################## TERMS OF USE #################################################################
# The following code is provided for demonstration purpose only, and should not be used without independent verification. Recorded Future   #
# makes no representations or warranties, express, implied, statutory, or otherwise, regarding this code, and provides it strictly "as-is". #
# Recorded Future shall not be liable for, and you assume all risk of using the foregoing.                                                  #
#############################################################################################################################################

import requests
import json
import datetime
import re


# !!!! CHECK TOGGLE ON DEBUG MODE TO RUN

def queryS1():

    # Set the API endpoint and your API key
    url = "https:///web/api/v2.1/threats" #enter full URL here such as https://domain.com/web/api/v2.1/threats
    api_key = "" #enter API key here
    # Set the headers for the API request
    headers = {
        "Content-Type": "application/json",
        "Accept": "application/json",
        "Authorization": "ApiToken {}".format(api_key)
    }
    # Set the query parameters for the API request

    Previous_Date = datetime.datetime.today() - datetime.timedelta(days=1)
    # print ('Previous Date: ' + str(Previous_Date))
    nextCursor = None

    params = {
        #    "status": "active"
        "createdAt__gt": Previous_Date,
        "cursor": nextCursor
    }
    # Make the API request
    
    json_object = []
    data = []
    while True:
        if nextCursor:
                params['cursor'] = nextCursor
        response = requests.get(url, headers=headers, params=params)
        json_object = response.json()
        if isinstance(json_object['data'], list):
                data.extend(json_object['data'])
        else:
                data.append(json_object['data'])
        nextCursor = json_object.get('pagination', {}).get('nextCursor', None)

        if not nextCursor:
                break
    
    #print(data)

    # define list to iterate IOCs
    ioc_list = []
    # define submit options at start of json data payload in response
    json_start = dict(options=dict(debug=bool(True), summary=bool(True)))


    # Start to store variables to build json object to be uploaded to Collective Insights
    for q in data:
        print('>>>>>>>>>>>>>>>>>>')
        print(json.dumps((q),indent=1))
        timestamp = q['threatInfo']['createdAt']
        ioc = q['threatInfo']['sha1']
        threatId = q['threatInfo']['threatId']

        # separate out the indicators json object and regex for Mitre Techniques if available
        mitre_codes = []
        mitre_finder= ""
        # mitre_codes.append("T1211")
        for item in q['indicators']:
            try:
                mitre_finder = re.findall(r"name': '(T[0-9\.]+)'", str(item), flags=re.MULTILINE | re.DOTALL)[0]
            # print(mitre_code)
            except:
                mitre_finder=""
            mitre_codes.append(mitre_finder)

        # format json per the needs of https://api.recordedfuture.com/collective-insights
        jsonBuilder = {
            'timestamp': timestamp,
            'ioc': {
                "type": "hash",
                "value": ioc
            },
            'incident': {
                "id": threatId,
                "name": "SentinelOne Threat Detection",
                "type": "sentinelOne-threat-detection"
            },
            "mitre_codes":
            mitre_codes,
            "detection" : { "type" : "correlation",
                            "sub_type" : "" }
        }
        ioc_list.append(jsonBuilder)
    # build json payload as dict
    uploadJson = dict(json_start, data=ioc_list)
    # pretty print payload as json
    print(json.dumps((uploadJson),indent=1))
    # count submissions
    uploadCount = len(ioc_list)
    return uploadCount, uploadJson


def submitCollectiveInsights(collective_insights_upload):
    url = "https://api.recordedfuture.com/collective-insights/detections"
    api_key = "" # API token from Recorded Future for collective Insights API
    # Set the headers for the API request
    headers = {
        "Content-Type": "application/json",
        "Accept": "application/json",
        "X-RFToken": "{}".format(api_key)
    }

    # assign data variable to the json payload submitted to this function
    data = collective_insights_upload
    print(data)

    # post data to collective insights using arguments created above
    response = requests.post(url, headers=headers, data=json.dumps(data))
    # print(response)
    return response


try:
    resultCount, collective_insights_upload = queryS1()
    print("Ran without Error - Generated " + str(resultCount) + " results")
    submitPayload = submitCollectiveInsights(collective_insights_upload)
    print('=======')
    print(submitPayload.text)
except requests.exceptions.RequestException as e:  # This is the correct syntax
    raise SystemExit(e)