|
17 | 17 |
|
18 | 18 | import sys |
19 | 19 | import warnings |
20 | | -from typing import Any, Callable, NamedTuple, List, Optional, TYPE_CHECKING |
| 20 | +from typing import Any, Callable, Dict, NamedTuple, List, Optional, TYPE_CHECKING |
21 | 21 |
|
22 | 22 | from pyspark.errors import PySparkTypeError |
23 | 23 | from pyspark.storagelevel import StorageLevel |
@@ -77,6 +77,15 @@ class Function(NamedTuple): |
77 | 77 | isTemporary: bool |
78 | 78 |
|
79 | 79 |
|
| 80 | +class CachedTable(NamedTuple): |
| 81 | + name: str |
| 82 | + storageLevel: str |
| 83 | + |
| 84 | + |
| 85 | +class CatalogTablePartition(NamedTuple): |
| 86 | + partition: str |
| 87 | + |
| 88 | + |
80 | 89 | class Catalog: |
81 | 90 | """User-facing catalog API, accessible through `SparkSession.catalog`. |
82 | 91 |
|
@@ -161,6 +170,127 @@ def listCatalogs(self, pattern: Optional[str] = None) -> List[CatalogMetadata]: |
161 | 170 | ) |
162 | 171 | return catalogs |
163 | 172 |
|
| 173 | + def listCachedTables(self) -> List[CachedTable]: |
| 174 | + """Lists named in-memory cache entries (same as ``SHOW CACHED TABLES``). |
| 175 | +
|
| 176 | + .. versionadded:: 4.2.0 |
| 177 | + """ |
| 178 | + iter = self._jcatalog.listCachedTables().toLocalIterator() |
| 179 | + out: List[CachedTable] = [] |
| 180 | + while iter.hasNext(): |
| 181 | + j = iter.next() |
| 182 | + out.append(CachedTable(name=j.name(), storageLevel=j.storageLevel())) |
| 183 | + return out |
| 184 | + |
| 185 | + def dropTable(self, tableName: str, ifExists: bool = False, purge: bool = False) -> None: |
| 186 | + """Drops a persistent table. |
| 187 | +
|
| 188 | + .. versionadded:: 4.2.0 |
| 189 | + """ |
| 190 | + self._jcatalog.dropTable(tableName, ifExists, purge) |
| 191 | + |
| 192 | + def dropView(self, viewName: str, ifExists: bool = False) -> None: |
| 193 | + """Drops a persistent view. |
| 194 | +
|
| 195 | + .. versionadded:: 4.2.0 |
| 196 | + """ |
| 197 | + self._jcatalog.dropView(viewName, ifExists) |
| 198 | + |
| 199 | + def createDatabase( |
| 200 | + self, dbName: str, ifNotExists: bool = False, properties: Optional[Dict[str, str]] = None |
| 201 | + ) -> None: |
| 202 | + """Creates a namespace (database). |
| 203 | +
|
| 204 | + .. versionadded:: 4.2.0 |
| 205 | + """ |
| 206 | + ju = self._sc._gateway.jvm.java.util |
| 207 | + m = ju.HashMap() |
| 208 | + if properties: |
| 209 | + for k, v in properties.items(): |
| 210 | + m.put(k, v) |
| 211 | + self._jcatalog.createDatabase(dbName, ifNotExists, m) |
| 212 | + |
| 213 | + def dropDatabase(self, dbName: str, ifExists: bool = False, cascade: bool = False) -> None: |
| 214 | + """Drops a namespace. |
| 215 | +
|
| 216 | + .. versionadded:: 4.2.0 |
| 217 | + """ |
| 218 | + self._jcatalog.dropDatabase(dbName, ifExists, cascade) |
| 219 | + |
| 220 | + def listPartitions(self, tableName: str) -> List[CatalogTablePartition]: |
| 221 | + """Lists partitions (same as ``SHOW PARTITIONS``). |
| 222 | +
|
| 223 | + .. versionadded:: 4.2.0 |
| 224 | + """ |
| 225 | + iter = self._jcatalog.listPartitions(tableName).toLocalIterator() |
| 226 | + out: List[CatalogTablePartition] = [] |
| 227 | + while iter.hasNext(): |
| 228 | + j = iter.next() |
| 229 | + out.append(CatalogTablePartition(partition=j.partition())) |
| 230 | + return out |
| 231 | + |
| 232 | + def listViews(self, dbName: Optional[str] = None, pattern: Optional[str] = None) -> List[Table]: |
| 233 | + """Lists views in a namespace. |
| 234 | +
|
| 235 | + .. versionadded:: 4.2.0 |
| 236 | + """ |
| 237 | + if pattern is not None and dbName is None: |
| 238 | + dbName = self.currentDatabase() |
| 239 | + if dbName is None: |
| 240 | + iter = self._jcatalog.listViews().toLocalIterator() |
| 241 | + elif pattern is None: |
| 242 | + iter = self._jcatalog.listViews(dbName).toLocalIterator() |
| 243 | + else: |
| 244 | + iter = self._jcatalog.listViews(dbName, pattern).toLocalIterator() |
| 245 | + views = [] |
| 246 | + while iter.hasNext(): |
| 247 | + jtable = iter.next() |
| 248 | + jnamespace = jtable.namespace() |
| 249 | + if jnamespace is not None: |
| 250 | + namespace = [jnamespace[i] for i in range(0, len(jnamespace))] |
| 251 | + else: |
| 252 | + namespace = None |
| 253 | + views.append( |
| 254 | + Table( |
| 255 | + name=jtable.name(), |
| 256 | + catalog=jtable.catalog(), |
| 257 | + namespace=namespace, |
| 258 | + description=jtable.description(), |
| 259 | + tableType=jtable.tableType(), |
| 260 | + isTemporary=jtable.isTemporary(), |
| 261 | + ) |
| 262 | + ) |
| 263 | + return views |
| 264 | + |
| 265 | + def getTableProperties(self, tableName: str) -> Dict[str, str]: |
| 266 | + """Returns table properties as a dict. |
| 267 | +
|
| 268 | + .. versionadded:: 4.2.0 |
| 269 | + """ |
| 270 | + jm = self._jcatalog.getTableProperties(tableName) |
| 271 | + return {k: jm.get(k) for k in jm.keySet()} |
| 272 | + |
| 273 | + def getCreateTableString(self, tableName: str, asSerde: bool = False) -> str: |
| 274 | + """Returns ``SHOW CREATE TABLE`` DDL for a relation. |
| 275 | +
|
| 276 | + .. versionadded:: 4.2.0 |
| 277 | + """ |
| 278 | + return self._jcatalog.getCreateTableString(tableName, asSerde) |
| 279 | + |
| 280 | + def truncateTable(self, tableName: str) -> None: |
| 281 | + """Truncates a table. |
| 282 | +
|
| 283 | + .. versionadded:: 4.2.0 |
| 284 | + """ |
| 285 | + self._jcatalog.truncateTable(tableName) |
| 286 | + |
| 287 | + def analyzeTable(self, tableName: str, noScan: bool = False) -> None: |
| 288 | + """Runs ``ANALYZE TABLE ... COMPUTE STATISTICS``. |
| 289 | +
|
| 290 | + .. versionadded:: 4.2.0 |
| 291 | + """ |
| 292 | + self._jcatalog.analyzeTable(tableName, noScan) |
| 293 | + |
164 | 294 | def currentDatabase(self) -> str: |
165 | 295 | """ |
166 | 296 | Returns the current default database in this session. |
|
0 commit comments