MCPcopy Create free account
hub / github.com/IBM/Project_CodeNet / main

Function main

tools/tokenizer/tokenize.c:101–455  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

99#include "libtoken.h"
100
101int main(int argc, char *argv[])
102{
103 extern char *optarg;
104 extern int opterr;
105 extern int optind;
106 int option;
107 char const *opt_str = "1acdhjkl:m:nNo:rsvwW";
108 char usage_str[80];
109
110 const char *token;
111 enum TokenClass type;
112 unsigned line;
113 unsigned col;
114 unsigned pos;
115 unsigned token_len;
116 unsigned num_files = 0; // number of files read
117 int start_token = 0; // when 1 start filename pseudo-token
118 int continuous_files = 0; // when 1 do not reset after each file
119
120 char *outfile = 0;
121 enum { PLAIN, CSV, JSON, JSONL, XML, RAW } mode = PLAIN;
122 int first_time = 1;
123 Language source;
124 int explicit_source = 0;
125 int append = 0;
126 int suppress_newline = 0;
127
128 sprintf(usage_str, "usage: %%s [ -%s ] [ FILES ]\n", opt_str);
129
130 /* Process arguments: */
131 while ((option = getopt(argc, argv, opt_str)) != EOF) {
132 switch (option) {
133
134 case '1':
135 continuous_files = 1;
136 break;
137
138 case 'a':
139 append = 1;
140 break;
141
142 case 'c':
143 hash_as_comment = 1;
144 break;
145
146 case 'd':
147 debug = verbose = 1;
148 break;
149
150 case 'h':
151fputs(
152"A tokenizer for C/C++ (and Java) source code with output in 6 formats.\n"
153"Recognizes the following token classes: keyword, identifier, integer,\n"
154"floating, string, character, operator, and preprocessor.\n\n", stderr);
155fprintf(stderr, usage_str, basename(argv[0]));
156fputs(
157"\nCommand line options are:\n"
158"-a : append to output file instead of create or overwrite.\n"

Callers

nothing calls this directly

Calls 5

set_or_detect_langFunction · 0.85
C_tokenize_intFunction · 0.85
XML_escapeFunction · 0.70
CSV_escapeFunction · 0.70
JSON_escapeFunction · 0.70

Tested by

no test coverage detected