Functions — github.com/commoncrawl/cc-index-table

hub / github.com/commoncrawl/cc-index-table / functions

Functions160 in github.com/commoncrawl/cc-index-table

⨍ Functions 160◇ Types & classes 27

↓ 18 callersMethodgetHostName

()

src/main/java/org/commoncrawl/net/WarcUri.java:114

↓ 16 callersMethodgetHostName

(String url)

src/test/java/org/commoncrawl/net/TestURL.java:70

↓ 15 callersMethodgetString

(String key)

src/main/java/org/commoncrawl/spark/IndexTable.java:134

↓ 13 callersMethodgetHostName

()

src/main/java/org/commoncrawl/net/HostName.java:179

↓ 10 callersMethodgetDomainNameUnderRegistrySuffix

()

src/main/java/org/commoncrawl/net/HostName.java:187

↓ 10 callersMethodtoString

()

src/main/java/org/commoncrawl/net/WarcUri.java:97

↓ 9 callersFunction_create_mock_parquet_file

(column_name: str, row_groups_stats: list[tuple[str, str]])

src/util/test/test_are_part_min_max_increasing.py:7

↓ 8 callersFunctionare_parquet_file_row_groups_min_max_ordered

(pf: pq.ParquetFile, column_name: str)

src/util/are_part_min_max_increasing.py:14

↓ 8 callersMethodgetReverseHost

()

src/main/java/org/commoncrawl/net/HostName.java:199

↓ 7 callersMethodgetPath

()

src/main/java/org/commoncrawl/net/WarcUri.java:131

↓ 7 callersMethodgetPrivateDomainName

()

src/main/java/org/commoncrawl/net/HostName.java:195

↓ 7 callersMethodgetProtocol

()

src/main/java/org/commoncrawl/net/WarcUri.java:118

↓ 7 callersMethodnormalizeMalformedHttpSlashes

(String uriString)

src/main/java/org/commoncrawl/net/WarcUri.java:73

↓ 7 callersMethodreadJsonSchemaResource

(String resource)

src/main/java/org/commoncrawl/spark/IndexTable.java:367

↓ 6 callersMethodgetParts

(String filename)

src/main/java/org/commoncrawl/spark/util/CCWarcFilenameParser.java:65

↓ 6 callersMethodgetPort

()

src/main/java/org/commoncrawl/net/WarcUri.java:122

↓ 6 callersMethodgetQuery

()

src/main/java/org/commoncrawl/net/WarcUri.java:138

↓ 5 callersMethodgetInt

(String key)

src/main/java/org/commoncrawl/spark/IndexTable.java:141

↓ 5 callersMethodgetReverseHostPart

(int i)

src/main/java/org/commoncrawl/net/HostName.java:203

↓ 5 callersMethodsetHostName

(String name)

src/main/java/org/commoncrawl/net/HostName.java:119

↓ 5 callersMethodtestSingleRow

(Row row, StructType schema)

src/test/java/org/commoncrawl/spark/TestIndexTableBase.java:45

↓ 4 callersMethodgetRegistrySuffix

()

src/main/java/org/commoncrawl/net/HostName.java:183

↓ 4 callersMethodgetUrlString

()

src/main/java/org/commoncrawl/net/WarcUri.java:110

↓ 4 callersMethodwrite

(Text key, byte[] value)

src/main/java/org/commoncrawl/spark/util/WarcRecordWriter.java:142

↓ 3 callersFunctionare_all_parts_min_max_ordered

(file_or_s3_url_list: list[str], sort_column_name: str)

src/util/are_part_min_max_increasing.py:37

↓ 3 callersMethodasRow

Create {@link Row} representing a host (data type "string" if not otherwise specified): <ol> <li>host name <li>parts 1 - 5 of the reversed h

src/main/java/org/commoncrawl/net/HostName.java:268

↓ 3 callersMethodgetHostNameReversed

()

src/main/java/org/commoncrawl/net/HostName.java:207

↓ 3 callersMethodgetHttpStatus

(String fromKey)

src/main/java/org/commoncrawl/spark/IndexTable.java:198

↓ 3 callersMethodgetLong

(String key)

src/main/java/org/commoncrawl/spark/IndexTable.java:148

↓ 3 callersMethodgetScheme

()

src/main/java/org/commoncrawl/net/WarcUri.java:93

↓ 3 callersMethodgetWarcUri

(String fromKey)

src/main/java/org/commoncrawl/spark/IndexTable.java:190

↓ 3 callersMethodhelp

(Options options)

src/main/java/org/commoncrawl/spark/IndexTable.java:432

↓ 3 callersMethodhelp

(Options options)

src/main/java/org/commoncrawl/spark/examples/CCIndexExport.java:89

↓ 3 callersMethodrun

(String inputPaths, String outputPath, Function<String, Row> mapIndexEntries)

src/main/java/org/commoncrawl/spark/IndexTable.java:384

↓ 3 callersMethodtestField

(String expectedValue, String fieldName, Row row, StructType schema)

src/test/java/org/commoncrawl/spark/TestIndexTableBase.java:94

↓ 3 callersMethodtestSingleCdxLine

(String cdxLine, String schemaFile)

src/test/java/org/commoncrawl/spark/TestIndexTable.java:33

↓ 2 callersMethodconvertCdxLine

(String line)

src/main/java/org/commoncrawl/spark/CCIndex2Table.java:83

↓ 2 callersMethodconvertCdxLine

(String line)

src/main/java/org/commoncrawl/spark/EOTIndexTable.java:71

↓ 2 callersMethodexecuteQuery

(SparkSession spark, String sqlQuery)

src/main/java/org/commoncrawl/spark/examples/CCIndexExport.java:70

↓ 2 callersMethodgetCdxLine

()

src/test/java/org/commoncrawl/spark/TestIndexTable.java:29

↓ 2 callersMethodgetCdxLine

()

src/test/java/org/commoncrawl/spark/TestCCIndex2Table.java:26

↓ 2 callersMethodgetFieldIndex

(String name, StructType schema)

src/test/java/org/commoncrawl/spark/TestIndexTableBase.java:89

↓ 2 callersMethodgetPrivateSuffix

()

src/main/java/org/commoncrawl/net/HostName.java:191

↓ 2 callersMethodgetShort

(String key)

src/main/java/org/commoncrawl/spark/IndexTable.java:155

↓ 2 callersMethodloadTable

(SparkSession spark, String tablePath, String tableName)

src/main/java/org/commoncrawl/spark/examples/CCIndexExport.java:64

↓ 2 callersMethodparseAndSetURI

(String uriString)

src/main/java/org/commoncrawl/net/WarcUri.java:58

↓ 2 callersMethodprettyPrintBytes

(long value)

src/main/java/org/commoncrawl/spark/JobStatsListener.java:50

↓ 2 callersMethodreport

()

src/main/java/org/commoncrawl/spark/JobStatsListener.java:62

↓ 1 callersMethodaddCommandLineOptions

Add command-line options. Subclasses may override the method to add their options but should call <code>super.addCommandLineOptions(options)</code>.

src/main/java/org/commoncrawl/spark/IndexTable.java:453

↓ 1 callersMethodaddOptions

()

src/main/java/org/commoncrawl/spark/examples/CCIndexExport.java:102

↓ 1 callersMethodapplyCommandLineOptions

Apply command-line options. Subclasses may override the method in order to apply their options but should call <code>super.applyCommandLineOptions(cli

src/main/java/org/commoncrawl/spark/IndexTable.java:479

↓ 1 callersMethodcanonicalizeIpAddress

Canonicalize IP address strings which are accepted by {@link InetAddress#getByName(String)}. @param ipAddrStr string representing a <strong>valid</st

src/main/java/org/commoncrawl/net/HostName.java:237

↓ 1 callersMethodclose

(TaskAttemptContext context)

src/main/java/org/commoncrawl/spark/util/WarcRecordWriter.java:147

↓ 1 callersMethodconvertCdxLine

Convert a CDX line into a table row. See {@link CdxLine} for the CDX line format. The JSON object in the CDX line must include a key "url"

src/main/java/org/commoncrawl/spark/IndexTable.java:253

↓ 1 callersMethodgetBoolean

(String key)

src/main/java/org/commoncrawl/spark/IndexTable.java:169

↓ 1 callersMethodgetByte

(String key)

src/main/java/org/commoncrawl/spark/IndexTable.java:162

↓ 1 callersMethodgetCCWarcRecord

(S3Client s3, String filename, int offset, int length)

src/main/java/org/commoncrawl/spark/examples/CCIndexWarcExport.java:131

↓ 1 callersMethodgetCdxLine

()

src/test/java/org/commoncrawl/spark/TestEOTIndexTable.java:27

↓ 1 callersMethodgetCdxLineDNSRecord

()

src/test/java/org/commoncrawl/spark/TestEOTIndexTable.java:31

↓ 1 callersMethodgetDouble

(String key)

src/main/java/org/commoncrawl/spark/IndexTable.java:183

↓ 1 callersMethodgetFileName

Compose a unique WARC file name. @param prefix WARC file name prefix @param date creation date @param partition partition ID @return (unique)

src/main/java/org/commoncrawl/spark/util/WarcRecordWriter.java:135

↓ 1 callersMethodgetFloat

(String key)

src/main/java/org/commoncrawl/spark/IndexTable.java:176

↓ 1 callersMethodgetSession

()

src/test/java/org/commoncrawl/spark/TestIndexTableBase.java:41

↓ 1 callersFunctionis_gzip

(content: bytes)

src/util/are_part_min_max_increasing.py:55

↓ 1 callersFunctionmain

(args)

src/util/athena_query_multiple_crawls.py:69

↓ 1 callersMethodparseOptions

(String[] args, List<String> arguments)

src/main/java/org/commoncrawl/spark/examples/CCIndexExport.java:117

↓ 1 callersFunctionparse_arguments

(args)

src/util/athena_query_multiple_crawls.py:47

↓ 1 callersFunctionquery_execute

(crawl, cursor, args)

src/util/athena_query_multiple_crawls.py:25

↓ 1 callersFunctionread_file_list

(path_or_url: str, prefix: str)

src/util/are_part_min_max_increasing.py:59

↓ 1 callersMethodreverseHost

Split host name into parts in reverse order: <code>www.example.com</code> becomes <code>[com, example, www]</code>. @param hostName host name, e.g. <

src/main/java/org/commoncrawl/net/HostName.java:220

↓ 1 callersMethodrun

(String tablePath, String outputPath)

src/main/java/org/commoncrawl/spark/examples/CCIndexWarcExport.java:154

↓ 1 callersMethodrun

(String tablePath, String outputPath)

src/main/java/org/commoncrawl/spark/examples/CCIndexExport.java:77

↓ 1 callersMethodvalidateField

(int i, Object val, StructField field, String indentation)

src/test/java/org/commoncrawl/spark/TestIndexTableBase.java:70

↓ 1 callersMethodvalidateSchemaFirstRow

(Dataset<Row> df, StructType schema)

src/test/java/org/commoncrawl/spark/TestIndexTableBase.java:54

↓ 1 callersMethodvalidateSchemaRow

(Row row, StructType schema)

src/test/java/org/commoncrawl/spark/TestIndexTableBase.java:62

MethodCdxLine

(String line)

src/main/java/org/commoncrawl/spark/CCIndex2Table.java:53

MethodCdxLine

(String line)

src/main/java/org/commoncrawl/spark/EOTIndexTable.java:47

MethodCdxLine

(String line)

src/main/java/org/commoncrawl/spark/IndexTable.java:109

MethodFilenameParseError

(String message)

src/main/java/org/commoncrawl/spark/util/CCWarcFilenameParser.java:60

MethodFilenameParts

(String crawl, String segment, String subset)

src/main/java/org/commoncrawl/spark/util/CCWarcFilenameParser.java:48

MethodHostName

(String hostName)

src/main/java/org/commoncrawl/net/HostName.java:79

MethodWarcRecordWriter

(Configuration conf, Path outputPath, int partition, TaskAttemptContext context)

src/main/java/org/commoncrawl/spark/util/WarcRecordWriter.java:55

MethodWarcUri

(String uriString)

src/main/java/org/commoncrawl/net/WarcUri.java:40

MethodabortTask

(TaskAttemptContext taskContext)

src/main/java/org/commoncrawl/spark/util/NullOutputCommitter.java:44

MethodaddCommandLineOptions

(Options options)

src/main/java/org/commoncrawl/spark/CCIndex2Table.java:156

MethodaddOptions

()

src/main/java/org/commoncrawl/spark/examples/CCIndexWarcExport.java:65

MethodapplyCommandLineOptions

(CommandLine cli)

src/main/java/org/commoncrawl/spark/CCIndex2Table.java:165

MethodapplyCommandLineOptions

(CommandLine cli)

src/main/java/org/commoncrawl/spark/EOTIndexTable.java:117

MethodcheckOutputSpecs

(JobContext job)

src/main/java/org/commoncrawl/spark/util/WarcFileOutputFormat.java:89

MethodcommitTask

(TaskAttemptContext taskContext)

src/main/java/org/commoncrawl/spark/util/NullOutputCommitter.java:40

MethodgetHostNameMalformedHttpShouldNotBeEmpty

()

src/test/java/org/commoncrawl/net/WarcUriTest.java:35

MethodgetHostNameValidHttpHostShouldNotBeEmpty

()

src/test/java/org/commoncrawl/net/WarcUriTest.java:44

MethodgetHostNameValidHttpsHostShouldNotBeEmpty

()

src/test/java/org/commoncrawl/net/WarcUriTest.java:53

MethodgetHostNameWithMalformedHttpsShouldNotBeEmpty

()

src/test/java/org/commoncrawl/net/WarcUriTest.java:26

MethodgetOutputCommitter

(TaskAttemptContext context)

src/main/java/org/commoncrawl/spark/util/WarcFileOutputFormat.java:66

MethodgetRecordWriter

(TaskAttemptContext context)

src/main/java/org/commoncrawl/spark/util/WarcFileOutputFormat.java:47

MethodgetURI

()

src/main/java/org/commoncrawl/net/WarcUri.java:106

MethodgetURL

()

src/main/java/org/commoncrawl/net/WarcUri.java:102

Methodmain

(String[] args)

src/main/java/org/commoncrawl/spark/CCIndex2Table.java:191

Methodmain

(String[] args)

src/main/java/org/commoncrawl/spark/EOTIndexTable.java:130

next →1–100 of 160, ranked by callers